commit 0565cd461dffea07727475aef140e162a7bc29c2 Author: ModelHub XC Date: Thu Oct 22 13:41:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Jeesup/svd-safety-l2_base_k0_a1p0_free_remove30 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..51089e2 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,126 @@ +LLAMA 2 COMMUNITY LICENSE AGREEMENT +Llama 2 Version Release Date: July 18, 2023 + +"Agreement" means the terms and conditions for use, reproduction, distribution and +modification of the Llama Materials set forth herein. + +"Documentation" means the specifications, manuals and documentation +accompanying Llama 2 distributed by Meta at ai.meta.com/resources/models-and- +libraries/llama-downloads/. + +"Licensee" or "you" means you, or your employer or any other person or entity (if +you are entering into this Agreement on such person or entity's behalf), of the age +required under applicable laws, rules or regulations to provide legal consent and that +has legal authority to bind your employer or such other person or entity if you are +entering in this Agreement on their behalf. + +"Llama 2" means the foundational large language models and software and +algorithms, including machine-learning model code, trained model weights, +inference-enabling code, training-enabling code, fine-tuning enabling code and other +elements of the foregoing distributed by Meta at ai.meta.com/resources/models-and- +libraries/llama-downloads/. + +"Llama Materials" means, collectively, Meta's proprietary Llama 2 and +Documentation (and any portion thereof) made available under this Agreement. + +"Meta" or "we" means Meta Platforms Ireland Limited (if you are located in or, if you +are an entity, your principal place of business is in the EEA or Switzerland) and Meta +Platforms, Inc. (if you are located outside of the EEA or Switzerland). + +By clicking "I Accept" below or by using or distributing any portion or element of the +Llama Materials, you agree to be bound by this Agreement. + +1. License Rights and Redistribution. + + a. Grant of Rights. You are granted a non-exclusive, worldwide, non- +transferable and royalty-free limited license under Meta's intellectual property or +other rights owned by Meta embodied in the Llama Materials to use, reproduce, +distribute, copy, create derivative works of, and make modifications to the Llama +Materials. + + b. Redistribution and Use. + + i. If you distribute or make the Llama Materials, or any derivative works +thereof, available to a third party, you shall provide a copy of this Agreement to such +third party. + ii. If you receive Llama Materials, or any derivative works thereof, from +a Licensee as part of an integrated end user product, then Section 2 of this +Agreement will not apply to you. + + iii. You must retain in all copies of the Llama Materials that you +distribute the following attribution notice within a "Notice" text file distributed as a +part of such copies: "Llama 2 is licensed under the LLAMA 2 Community License, +Copyright (c) Meta Platforms, Inc. All Rights Reserved." + + iv. Your use of the Llama Materials must comply with applicable laws +and regulations (including trade compliance laws and regulations) and adhere to the +Acceptable Use Policy for the Llama Materials (available at +https://ai.meta.com/llama/use-policy), which is hereby incorporated by reference into +this Agreement. + + v. You will not use the Llama Materials or any output or results of the +Llama Materials to improve any other large language model (excluding Llama 2 or +derivative works thereof). + +2. Additional Commercial Terms. If, on the Llama 2 version release date, the +monthly active users of the products or services made available by or for Licensee, +or Licensee's affiliates, is greater than 700 million monthly active users in the +preceding calendar month, you must request a license from Meta, which Meta may +grant to you in its sole discretion, and you are not authorized to exercise any of the +rights under this Agreement unless or until Meta otherwise expressly grants you +such rights. + +3. Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE +LLAMA MATERIALS AND ANY OUTPUT AND RESULTS THEREFROM ARE +PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY +WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR +FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE +FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING +THE LLAMA MATERIALS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR +USE OF THE LLAMA MATERIALS AND ANY OUTPUT AND RESULTS. + +4. Limitation of Liability. IN NO EVENT WILL META OR ITS AFFILIATES BE +LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, +NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS +AGREEMENT, FOR ANY LOST PROFITS OR ANY INDIRECT, SPECIAL, +CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN +IF META OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF +ANY OF THE FOREGOING. + +5. Intellectual Property. + + a. No trademark licenses are granted under this Agreement, and in +connection with the Llama Materials, neither Meta nor Licensee may use any name +or mark owned by or associated with the other or any of its affiliates, except as +required for reasonable and customary use in describing and redistributing the +Llama Materials. + + b. Subject to Meta's ownership of Llama Materials and derivatives made by or +for Meta, with respect to any derivative works and modifications of the Llama +Materials that are made by you, as between you and Meta, you are and will be the +owner of such derivative works and modifications. + + c. If you institute litigation or other proceedings against Meta or any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the Llama +Materials or Llama 2 outputs or results, or any portion of any of the foregoing, +constitutes infringement of intellectual property or other rights owned or licensable +by you, then any licenses granted to you under this Agreement shall terminate as of +the date such litigation or claim is filed or instituted. You will indemnify and hold +harmless Meta from and against any claim by any third party arising out of or related +to your use or distribution of the Llama Materials. + +6. Term and Termination. The term of this Agreement will commence upon your +acceptance of this Agreement or access to the Llama Materials and will continue in +full force and effect until terminated in accordance with the terms and conditions +herein. Meta may terminate this Agreement if you are in breach of any term or +condition of this Agreement. Upon termination of this Agreement, you shall delete +and cease use of the Llama Materials. Sections 3, 4 and 7 shall survive the +termination of this Agreement. + +7. Governing Law and Jurisdiction. This Agreement will be governed and +construed under the laws of the State of California without regard to choice of law +principles, and the UN Convention on Contracts for the International Sale of Goods +does not apply to this Agreement. The courts of California shall have exclusive +jurisdiction of any dispute arising out of this Agreement. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..5312edc --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +license: llama2 +base_model: meta-llama/Llama-2-7b-chat-hf +library_name: transformers +pipeline_tag: text-generation +tags: +- llama2 +- svd +- compression +- safety +- interpretability +--- + +# svd-safety-l2_base_k0_a1p0_free_remove30 + +A Llama-2-7b-chat checkpoint compressed with SVD-LLM to **70.0% of dense +parameters**, then given a **0.0% parameter budget** of restored SVD +components selected by the **`unknown`** rule. + +This is a research artifact from a study of how SVD compression damages safety +behaviour and which component-selection rule best repairs it. It is one cell of a +grid over selection rules and budgets; it is **not** a general-purpose chat model. + +## Provenance + +| field | value | +|---|---| +| base (uncompressed) | `meta-llama/Llama-2-7b-chat-hf` | +| compression | SVD-LLM, 30.00% of parameters removed | +| selection rule | `unknown` | +| restore budget | 0.000% of dense parameters | +| components restored | 0 | +| components swapped out | 0 | +| resulting parameter fraction | 0.6998 | +| seed | 42 | + +## Measured + +| metric | value | +|---|---| +| AdvBench ASR (HarmBench judge) | 0.0635 | +| StrongREJECT ASR (HarmBench judge) | 0.0511 | +| Macro over-refusal (WildGuard) | 0.2869 | +| WikiText-2 perplexity | 9.7848 | + + +## Intended use and limitations + +This checkpoint exists to measure safety/utility trade-offs under compression. +Several arms in the grid are **deliberately safety-degraded** relative to +Llama-2-7b-chat: compression alone raises attack-success rate, and the point of +the study is to quantify that and test recovery. Treat any given cell as an +experimental subject, not as a deployable assistant, and evaluate it yourself +before drawing conclusions from it. + +## Licence + +Llama 2 Community License. `LICENSE.txt` and `USE_POLICY.md` are included in this repository, and use of this derivative is bound by them. Built with Llama 2. diff --git a/USE_POLICY.md b/USE_POLICY.md new file mode 100644 index 0000000..abbcc19 --- /dev/null +++ b/USE_POLICY.md @@ -0,0 +1,50 @@ +# Llama 2 Acceptable Use Policy + +Meta is committed to promoting safe and fair use of its tools and features, including Llama 2. If you access or use Llama 2, you agree to this Acceptable Use Policy (“Policy”). The most recent copy of this policy can be found at [ai.meta.com/llama/use-policy](http://ai.meta.com/llama/use-policy). + +## Prohibited Uses +We want everyone to use Llama 2 safely and responsibly. You agree you will not use, or allow others to use, Llama 2 to: + +1. Violate the law or others’ rights, including to: + 1. Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content, such as: + 1. Violence or terrorism + 2. Exploitation or harm to children, including the solicitation, creation, acquisition, or dissemination of child exploitative content or failure to report Child Sexual Abuse Material + 3. Human trafficking, exploitation, and sexual violence + 4. The illegal distribution of information or materials to minors, including obscene materials, or failure to employ legally required age-gating in connection with such information or materials. + 5. Sexual solicitation + 6. Any other criminal activity + 2. Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals + 3. Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services + 4. Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices + 5. Collect, process, disclose, generate, or infer health, demographic, or other sensitive personal or private information about individuals without rights and consents required by applicable laws + 6. Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any products or services using the Llama 2 Materials + 7. Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system + + + +2. Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including use of Llama 2 related to the following: + 1. Military, warfare, nuclear industries or applications, espionage, use for materials or activities that are subject to the International Traffic Arms Regulations (ITAR) maintained by the United States Department of State + 2. Guns and illegal weapons (including weapon development) + 3. Illegal drugs and regulated/controlled substances + 4. Operation of critical infrastructure, transportation technologies, or heavy machinery + 5. Self-harm or harm to others, including suicide, cutting, and eating disorders + 6. Any content intended to incite or promote violence, abuse, or any infliction of bodily harm to an individual + + + +3. Intentionally deceive or mislead others, including use of Llama 2 related to the following: + 1. Generating, promoting, or furthering fraud or the creation or promotion of disinformation + 2. Generating, promoting, or furthering defamatory content, including the creation of defamatory statements, images, or other content + 3. Generating, promoting, or further distributing spam + 4. Impersonating another individual without consent, authorization, or legal right + 5. Representing that the use of Llama 2 or outputs are human-generated + 6. Generating or facilitating false online engagement, including fake reviews and other means of fake online engagement +4. Fail to appropriately disclose to end users any known dangers of your AI system + +Please report any violation of this Policy, software “bug,” or other problems that could lead to a violation of this Policy through one of the following means: + +* Reporting issues with the model: [github.com/facebookresearch/llama](http://github.com/facebookresearch/llama) +* Reporting risky content generated by the model: [developers.facebook.com/llama_output_feedback](http://developers.facebook.com/llama_output_feedback) +* Reporting bugs and security concerns: [facebook.com/whitehat/info](http://facebook.com/whitehat/info) +* Reporting violations of the Acceptable Use Policy or unlicensed uses of Llama: [LlamaUseReport@meta.com](mailto:LlamaUseReport@meta.com) + diff --git a/compression.json b/compression.json new file mode 100644 index 0000000..6efcd64 --- /dev/null +++ b/compression.json @@ -0,0 +1,2060 @@ +{ + "source": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", + "upstream": "AIoT-MLSys-Lab/SVD-LLM", + "stage": "finetuned", + "ratio_removed": 0.3, + "internal_ratio_kept": 0.7, + "from_pt": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove30/finetune_v/merge.pt", + "biases_verified": 0, + "mean_rel_err": 0.46960629166902174, + "max_rel_err": 1.2178252935409546, + "param_fraction": 0.6997563653659327, + "ranks": { + "0.q_proj": 1433, + "0.k_proj": 1433, + "0.v_proj": 1433, + "0.o_proj": 1433, + "0.gate_proj": 2089, + "0.up_proj": 2089, + "0.down_proj": 2089, + "1.q_proj": 1433, + "1.k_proj": 1433, + "1.v_proj": 1433, + "1.o_proj": 1433, + "1.gate_proj": 2089, + "1.up_proj": 2089, + "1.down_proj": 2089, + "2.q_proj": 1433, + "2.k_proj": 1433, + "2.v_proj": 1433, + "2.o_proj": 1433, + "2.gate_proj": 2089, + "2.up_proj": 2089, + "2.down_proj": 2089, + "3.q_proj": 1433, + "3.k_proj": 1433, + "3.v_proj": 1433, + "3.o_proj": 1433, + "3.gate_proj": 2089, + "3.up_proj": 2089, + "3.down_proj": 2089, + "4.q_proj": 1433, + "4.k_proj": 1433, + "4.v_proj": 1433, + "4.o_proj": 1433, + "4.gate_proj": 2089, + "4.up_proj": 2089, + "4.down_proj": 2089, + "5.q_proj": 1433, + "5.k_proj": 1433, + "5.v_proj": 1433, + "5.o_proj": 1433, + "5.gate_proj": 2089, + "5.up_proj": 2089, + "5.down_proj": 2089, + "6.q_proj": 1433, + "6.k_proj": 1433, + "6.v_proj": 1433, + "6.o_proj": 1433, + "6.gate_proj": 2089, + "6.up_proj": 2089, + "6.down_proj": 2089, + "7.q_proj": 1433, + "7.k_proj": 1433, + "7.v_proj": 1433, + "7.o_proj": 1433, + "7.gate_proj": 2089, + "7.up_proj": 2089, + "7.down_proj": 2089, + "8.q_proj": 1433, + "8.k_proj": 1433, + "8.v_proj": 1433, + "8.o_proj": 1433, + "8.gate_proj": 2089, + "8.up_proj": 2089, + "8.down_proj": 2089, + "9.q_proj": 1433, + "9.k_proj": 1433, + "9.v_proj": 1433, + "9.o_proj": 1433, + "9.gate_proj": 2089, + "9.up_proj": 2089, + "9.down_proj": 2089, + "10.q_proj": 1433, + "10.k_proj": 1433, + "10.v_proj": 1433, + "10.o_proj": 1433, + "10.gate_proj": 2089, + "10.up_proj": 2089, + "10.down_proj": 2089, + "11.q_proj": 1433, + "11.k_proj": 1433, + "11.v_proj": 1433, + "11.o_proj": 1433, + "11.gate_proj": 2089, + "11.up_proj": 2089, + "11.down_proj": 2089, + "12.q_proj": 1433, + "12.k_proj": 1433, + "12.v_proj": 1433, + "12.o_proj": 1433, + "12.gate_proj": 2089, + "12.up_proj": 2089, + "12.down_proj": 2089, + "13.q_proj": 1433, + "13.k_proj": 1433, + "13.v_proj": 1433, + "13.o_proj": 1433, + "13.gate_proj": 2089, + "13.up_proj": 2089, + "13.down_proj": 2089, + "14.q_proj": 1433, + "14.k_proj": 1433, + "14.v_proj": 1433, + "14.o_proj": 1433, + "14.gate_proj": 2089, + "14.up_proj": 2089, + "14.down_proj": 2089, + "15.q_proj": 1433, + "15.k_proj": 1433, + "15.v_proj": 1433, + "15.o_proj": 1433, + "15.gate_proj": 2089, + "15.up_proj": 2089, + "15.down_proj": 2089, + "16.q_proj": 1433, + "16.k_proj": 1433, + "16.v_proj": 1433, + "16.o_proj": 1433, + "16.gate_proj": 2089, + "16.up_proj": 2089, + "16.down_proj": 2089, + "17.q_proj": 1433, + "17.k_proj": 1433, + "17.v_proj": 1433, + "17.o_proj": 1433, + "17.gate_proj": 2089, + "17.up_proj": 2089, + "17.down_proj": 2089, + "18.q_proj": 1433, + "18.k_proj": 1433, + "18.v_proj": 1433, + "18.o_proj": 1433, + "18.gate_proj": 2089, + "18.up_proj": 2089, + "18.down_proj": 2089, + "19.q_proj": 1433, + "19.k_proj": 1433, + "19.v_proj": 1433, + "19.o_proj": 1433, + "19.gate_proj": 2089, + "19.up_proj": 2089, + "19.down_proj": 2089, + "20.q_proj": 1433, + "20.k_proj": 1433, + "20.v_proj": 1433, + "20.o_proj": 1433, + "20.gate_proj": 2089, + "20.up_proj": 2089, + "20.down_proj": 2089, + "21.q_proj": 1433, + "21.k_proj": 1433, + "21.v_proj": 1433, + "21.o_proj": 1433, + "21.gate_proj": 2089, + "21.up_proj": 2089, + "21.down_proj": 2089, + "22.q_proj": 1433, + "22.k_proj": 1433, + "22.v_proj": 1433, + "22.o_proj": 1433, + "22.gate_proj": 2089, + "22.up_proj": 2089, + "22.down_proj": 2089, + "23.q_proj": 1433, + "23.k_proj": 1433, + "23.v_proj": 1433, + "23.o_proj": 1433, + "23.gate_proj": 2089, + "23.up_proj": 2089, + "23.down_proj": 2089, + "24.q_proj": 1433, + "24.k_proj": 1433, + "24.v_proj": 1433, + "24.o_proj": 1433, + "24.gate_proj": 2089, + "24.up_proj": 2089, + "24.down_proj": 2089, + "25.q_proj": 1433, + "25.k_proj": 1433, + "25.v_proj": 1433, + "25.o_proj": 1433, + "25.gate_proj": 2089, + "25.up_proj": 2089, + "25.down_proj": 2089, + "26.q_proj": 1433, + "26.k_proj": 1433, + "26.v_proj": 1433, + "26.o_proj": 1433, + "26.gate_proj": 2089, + "26.up_proj": 2089, + "26.down_proj": 2089, + "27.q_proj": 1433, + "27.k_proj": 1433, + "27.v_proj": 1433, + "27.o_proj": 1433, + "27.gate_proj": 2089, + "27.up_proj": 2089, + "27.down_proj": 2089, + "28.q_proj": 1433, + "28.k_proj": 1433, + "28.v_proj": 1433, + "28.o_proj": 1433, + "28.gate_proj": 2089, + "28.up_proj": 2089, + "28.down_proj": 2089, + "29.q_proj": 1433, + "29.k_proj": 1433, + "29.v_proj": 1433, + "29.o_proj": 1433, + "29.gate_proj": 2089, + "29.up_proj": 2089, + "29.down_proj": 2089, + "30.q_proj": 1433, + "30.k_proj": 1433, + "30.v_proj": 1433, + "30.o_proj": 1433, + "30.gate_proj": 2089, + "30.up_proj": 2089, + "30.down_proj": 2089, + "31.q_proj": 1433, + "31.k_proj": 1433, + "31.v_proj": 1433, + "31.o_proj": 1433, + "31.gate_proj": 2089, + "31.up_proj": 2089, + "31.down_proj": 2089 + }, + "rel_err": { + "0.q_proj": 0.21154165267944336, + "0.k_proj": 0.25206056237220764, + "0.v_proj": 0.32299745082855225, + "0.o_proj": 0.32105162739753723, + "0.gate_proj": 0.3937569260597229, + "0.up_proj": 0.4039572477340698, + "0.down_proj": 0.4691493511199951, + "1.q_proj": 0.12650687992572784, + "1.k_proj": 0.10664405673742294, + "1.v_proj": 0.43515294790267944, + "1.o_proj": 0.3589252829551697, + "1.gate_proj": 0.37330377101898193, + "1.up_proj": 0.42149069905281067, + "1.down_proj": 0.5207186937332153, + "2.q_proj": 0.5791195034980774, + "2.k_proj": 1.2178252935409546, + "2.v_proj": 0.5660731196403503, + "2.o_proj": 0.5161605477333069, + "2.gate_proj": 0.4384504556655884, + "2.up_proj": 0.4704980254173279, + "2.down_proj": 0.5172862410545349, + "3.q_proj": 0.34828367829322815, + "3.k_proj": 0.3464677631855011, + "3.v_proj": 0.4843881130218506, + "3.o_proj": 0.5081186294555664, + "3.gate_proj": 0.4907146394252777, + "3.up_proj": 0.5106684565544128, + "3.down_proj": 0.48436760902404785, + "4.q_proj": 0.33908578753471375, + "4.k_proj": 0.341554194688797, + "4.v_proj": 0.4818229079246521, + "4.o_proj": 0.5117180943489075, + "4.gate_proj": 0.49236011505126953, + "4.up_proj": 0.517420768737793, + "4.down_proj": 0.4884204566478729, + "5.q_proj": 0.37434741854667664, + "5.k_proj": 0.3555288016796112, + "5.v_proj": 0.48474350571632385, + "5.o_proj": 0.5176644921302795, + "5.gate_proj": 0.4887177348136902, + "5.up_proj": 0.5189483761787415, + "5.down_proj": 0.48030030727386475, + "6.q_proj": 0.33946743607521057, + "6.k_proj": 0.3454365134239197, + "6.v_proj": 0.4700879752635956, + "6.o_proj": 0.4857877790927887, + "6.gate_proj": 0.483540415763855, + "6.up_proj": 0.5194035172462463, + "6.down_proj": 0.4795955717563629, + "7.q_proj": 0.3443571925163269, + "7.k_proj": 0.35092347860336304, + "7.v_proj": 0.46219494938850403, + "7.o_proj": 0.5031439661979675, + "7.gate_proj": 0.48110252618789673, + "7.up_proj": 0.519559919834137, + "7.down_proj": 0.4735674560070038, + "8.q_proj": 0.3535466492176056, + "8.k_proj": 0.36549273133277893, + "8.v_proj": 0.4496116638183594, + "8.o_proj": 0.5005570650100708, + "8.gate_proj": 0.47835803031921387, + "8.up_proj": 0.5146349668502808, + "8.down_proj": 0.4690413475036621, + "9.q_proj": 0.3716367483139038, + "9.k_proj": 0.37791308760643005, + "9.v_proj": 0.44794997572898865, + "9.o_proj": 0.503503143787384, + "9.gate_proj": 0.4749135673046112, + "9.up_proj": 0.509507954120636, + "9.down_proj": 0.46834155917167664, + "10.q_proj": 0.34816494584083557, + "10.k_proj": 0.35685667395591736, + "10.v_proj": 0.4425583481788635, + "10.o_proj": 0.4644082486629486, + "10.gate_proj": 0.4702244997024536, + "10.up_proj": 0.5053636431694031, + "10.down_proj": 0.4650056064128876, + "11.q_proj": 0.4666091799736023, + "11.k_proj": 0.5523228645324707, + "11.v_proj": 0.45560720562934875, + "11.o_proj": 0.4543610215187073, + "11.gate_proj": 0.4701629877090454, + "11.up_proj": 0.5056617259979248, + "11.down_proj": 0.4712400734424591, + "12.q_proj": 0.3658818304538727, + "12.k_proj": 0.3655855357646942, + "12.v_proj": 0.44657671451568604, + "12.o_proj": 0.4936954379081726, + "12.gate_proj": 0.47407642006874084, + "12.up_proj": 0.5060712695121765, + "12.down_proj": 0.4711635708808899, + "13.q_proj": 0.372084379196167, + "13.k_proj": 0.38780465722084045, + "13.v_proj": 0.4731096029281616, + "13.o_proj": 0.48436471819877625, + "13.gate_proj": 0.47704872488975525, + "13.up_proj": 0.5088900923728943, + "13.down_proj": 0.48266980051994324, + "14.q_proj": 0.35647156834602356, + "14.k_proj": 0.3611023724079132, + "14.v_proj": 0.4664199650287628, + "14.o_proj": 0.48168227076530457, + "14.gate_proj": 0.47688907384872437, + "14.up_proj": 0.5078426599502563, + "14.down_proj": 0.484318345785141, + "15.q_proj": 0.37352845072746277, + "15.k_proj": 0.37663161754608154, + "15.v_proj": 0.4720856249332428, + "15.o_proj": 0.49013736844062805, + "15.gate_proj": 0.4809935986995697, + "15.up_proj": 0.5118938088417053, + "15.down_proj": 0.4974278509616852, + "16.q_proj": 0.37138572335243225, + "16.k_proj": 0.375299334526062, + "16.v_proj": 0.49767112731933594, + "16.o_proj": 0.5181031227111816, + "16.gate_proj": 0.4802817106246948, + "16.up_proj": 0.5096001625061035, + "16.down_proj": 0.5094398260116577, + "17.q_proj": 0.40937861800193787, + "17.k_proj": 0.4079142212867737, + "17.v_proj": 0.5122692584991455, + "17.o_proj": 0.5284447073936462, + "17.gate_proj": 0.48317888379096985, + "17.up_proj": 0.5100035667419434, + "17.down_proj": 0.5069739818572998, + "18.q_proj": 0.43187329173088074, + "18.k_proj": 0.43729203939437866, + "18.v_proj": 0.5184658765792847, + "18.o_proj": 0.5557100772857666, + "18.gate_proj": 0.4912468194961548, + "18.up_proj": 0.5163496136665344, + "18.down_proj": 0.5074824690818787, + "19.q_proj": 0.4165549576282501, + "19.k_proj": 0.4135464131832123, + "19.v_proj": 0.5034193992614746, + "19.o_proj": 0.5436462163925171, + "19.gate_proj": 0.49490392208099365, + "19.up_proj": 0.5172055959701538, + "19.down_proj": 0.5076806545257568, + "20.q_proj": 0.409005731344223, + "20.k_proj": 0.40952786803245544, + "20.v_proj": 0.5161357522010803, + "20.o_proj": 0.5443912148475647, + "20.gate_proj": 0.4930191934108734, + "20.up_proj": 0.5144986510276794, + "20.down_proj": 0.5178073644638062, + "21.q_proj": 0.4219631850719452, + "21.k_proj": 0.42387834191322327, + "21.v_proj": 0.5112217664718628, + "21.o_proj": 0.5586152076721191, + "21.gate_proj": 0.4942625164985657, + "21.up_proj": 0.513978898525238, + "21.down_proj": 0.5138073563575745, + "22.q_proj": 0.4319232106208801, + "22.k_proj": 0.43579697608947754, + "22.v_proj": 0.5216929316520691, + "22.o_proj": 0.5557892918586731, + "22.gate_proj": 0.4942406117916107, + "22.up_proj": 0.51324063539505, + "22.down_proj": 0.51151043176651, + "23.q_proj": 0.46498385071754456, + "23.k_proj": 0.4636499881744385, + "23.v_proj": 0.5211161971092224, + "23.o_proj": 0.5465461015701294, + "23.gate_proj": 0.4975587725639343, + "23.up_proj": 0.5149858593940735, + "23.down_proj": 0.5041381120681763, + "24.q_proj": 0.4113546907901764, + "24.k_proj": 0.4162830710411072, + "24.v_proj": 0.5141201615333557, + "24.o_proj": 0.5522825717926025, + "24.gate_proj": 0.4978007376194, + "24.up_proj": 0.5156306624412537, + "24.down_proj": 0.4979228079319, + "25.q_proj": 0.4590507447719574, + "25.k_proj": 0.46130073070526123, + "25.v_proj": 0.5217674374580383, + "25.o_proj": 0.5457017421722412, + "25.gate_proj": 0.5010452270507812, + "25.up_proj": 0.5174975395202637, + "25.down_proj": 0.4916711151599884, + "26.q_proj": 0.4234485626220703, + "26.k_proj": 0.4319479763507843, + "26.v_proj": 0.5110377669334412, + "26.o_proj": 0.570513904094696, + "26.gate_proj": 0.5023332834243774, + "26.up_proj": 0.5195894837379456, + "26.down_proj": 0.4905470907688141, + "27.q_proj": 0.4701234698295593, + "27.k_proj": 0.4786141514778137, + "27.v_proj": 0.5294671058654785, + "27.o_proj": 0.5472278594970703, + "27.gate_proj": 0.5049896836280823, + "27.up_proj": 0.5212247967720032, + "27.down_proj": 0.48947373032569885, + "28.q_proj": 0.45211905241012573, + "28.k_proj": 0.4634537398815155, + "28.v_proj": 0.5256759524345398, + "28.o_proj": 0.5444328188896179, + "28.gate_proj": 0.5052045583724976, + "28.up_proj": 0.5199324488639832, + "28.down_proj": 0.49766820669174194, + "29.q_proj": 0.42195600271224976, + "29.k_proj": 0.42968735098838806, + "29.v_proj": 0.5174475908279419, + "29.o_proj": 0.5465685129165649, + "29.gate_proj": 0.5040810108184814, + "29.up_proj": 0.5160006880760193, + "29.down_proj": 0.5046414732933044, + "30.q_proj": 0.4532987177371979, + "30.k_proj": 0.45997172594070435, + "30.v_proj": 0.5314067602157593, + "30.o_proj": 0.5543007254600525, + "30.gate_proj": 0.4988415539264679, + "30.up_proj": 0.5041705369949341, + "30.down_proj": 0.5126928091049194, + "31.q_proj": 0.39961686730384827, + "31.k_proj": 0.40837952494621277, + "31.v_proj": 0.5121962428092957, + "31.o_proj": 0.5230640172958374, + "31.gate_proj": 0.4886501133441925, + "31.up_proj": 0.4899109899997711, + "31.down_proj": 0.5113798975944519 + }, + "whitening": { + "lam": 0.0, + "norm": "token", + "ridge": 0.0, + "token_mask": "nospecial", + "utility_cache": "/lustre/jeesup0103/svd-safety/work/whiten_cache/utility_wikitext256", + "utility_calib": "wikitext2:256", + "utility_tokens": 522240 + }, + "rank_by": "resmix", + "resmix_k": 0.0, + "resmix_k_abs": null, + "ranking": { + "n_matrices": 224, + "alpha": 1.0, + "k_s_total": 0, + "k_s_frac_of_rank": 0.0, + "utility_error_ratio": 1.0, + "resmix_k": 0.0, + "resmix_k_abs": null, + "resmix_alpha": 1.0 + }, + "ranking_per_matrix": { + "0.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39.388282935811475, + "T_cal": 39.388282935811475 + }, + "0.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 34.64168906039899, + "T_cal": 34.64168906039899 + }, + "0.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 2180.4545400725456, + "T_cal": 2180.4545400725456 + }, + "0.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 3.236008245712793, + "T_cal": 3.236008245712793 + }, + "0.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 233290.09868813131, + "T_cal": 233290.09868813131 + }, + "0.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 239977.84535169817, + "T_cal": 239977.84535169817 + }, + "0.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 4897.100459002034, + "T_cal": 4897.100459002034 + }, + "1.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 28729.716566027415, + "T_cal": 28729.716566027415 + }, + "1.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 21643.320093676906, + "T_cal": 21643.320093676906 + }, + "1.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 51962.35635288252, + "T_cal": 51962.35635288252 + }, + "1.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 467.6564444913811, + "T_cal": 467.6564444913811 + }, + "1.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 902488.3564397852, + "T_cal": 902488.3564397852 + }, + "1.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 1023626.5545441816, + "T_cal": 1023626.5545441816 + }, + "1.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 59103.256723517115, + "T_cal": 59103.256723517115 + }, + "2.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1774134.0302654633, + "T_cal": 1774134.0302654633 + }, + "2.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1778735.8435498467, + "T_cal": 1778735.8435498467 + }, + "2.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1784142.2006091198, + "T_cal": 1784142.2006091198 + }, + "2.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 24332.456126517485, + "T_cal": 24332.456126517485 + }, + "2.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 3897275.9891598956, + "T_cal": 3897275.9891598956 + }, + "2.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 3902478.238080211, + "T_cal": 3902478.238080211 + }, + "2.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 168321.99261231394, + "T_cal": 168321.99261231394 + }, + "3.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 12643629.215040559, + "T_cal": 12643629.215040559 + }, + "3.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 13716496.810724786, + "T_cal": 13716496.810724786 + }, + "3.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 7078173.434168732, + "T_cal": 7078173.434168732 + }, + "3.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39460.50467348984, + "T_cal": 39460.50467348984 + }, + "3.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 10688950.318662636, + "T_cal": 10688950.318662636 + }, + "3.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 9919457.723038245, + "T_cal": 9919457.723038245 + }, + "3.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 297727.46114792966, + "T_cal": 297727.46114792966 + }, + "4.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 12380687.063937424, + "T_cal": 12380687.063937424 + }, + "4.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 13230745.349890629, + "T_cal": 13230745.349890629 + }, + "4.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 7377412.447694868, + "T_cal": 7377412.447694868 + }, + "4.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 88498.14542927106, + "T_cal": 88498.14542927106 + }, + "4.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 15917695.579375152, + "T_cal": 15917695.579375152 + }, + "4.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 14251862.833006734, + "T_cal": 14251862.833006734 + }, + "4.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 548192.1997935446, + "T_cal": 548192.1997935446 + }, + "5.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 15545430.394920252, + "T_cal": 15545430.394920252 + }, + "5.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 16787557.251722403, + "T_cal": 16787557.251722403 + }, + "5.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 9192295.965431035, + "T_cal": 9192295.965431035 + }, + "5.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 116922.83781016959, + "T_cal": 116922.83781016959 + }, + "5.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 19895489.005969524, + "T_cal": 19895489.005969524 + }, + "5.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 17966035.376588047, + "T_cal": 17966035.376588047 + }, + "5.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 833079.9087956194, + "T_cal": 833079.9087956194 + }, + "6.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 21172200.32657173, + "T_cal": 21172200.32657173 + }, + "6.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 23004804.61069621, + "T_cal": 23004804.61069621 + }, + "6.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 12149843.699887238, + "T_cal": 12149843.699887238 + }, + "6.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 203828.19718049304, + "T_cal": 203828.19718049304 + }, + "6.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 25536093.872205686, + "T_cal": 25536093.872205686 + }, + "6.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 22666967.386803932, + "T_cal": 22666967.386803932 + }, + "6.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 1194725.368004628, + "T_cal": 1194725.368004628 + }, + "7.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 23996831.82645592, + "T_cal": 23996831.82645592 + }, + "7.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 25165174.584726967, + "T_cal": 25165174.584726967 + }, + "7.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 13360452.246222524, + "T_cal": 13360452.246222524 + }, + "7.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 286590.00475342566, + "T_cal": 286590.00475342566 + }, + "7.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 29828875.85005765, + "T_cal": 29828875.85005765 + }, + "7.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 26955563.39740563, + "T_cal": 26955563.39740563 + }, + "7.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 1598230.184621131, + "T_cal": 1598230.184621131 + }, + "8.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 25595703.640618827, + "T_cal": 25595703.640618827 + }, + "8.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 27773611.036283582, + "T_cal": 27773611.036283582 + }, + "8.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 13208919.58704463, + "T_cal": 13208919.58704463 + }, + "8.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 429615.15508663515, + "T_cal": 429615.15508663515 + }, + "8.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 30698682.080411024, + "T_cal": 30698682.080411024 + }, + "8.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 29097623.450163342, + "T_cal": 29097623.450163342 + }, + "8.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 1943908.4716851546, + "T_cal": 1943908.4716851546 + }, + "9.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 30298298.18147658, + "T_cal": 30298298.18147658 + }, + "9.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 32860451.102318216, + "T_cal": 32860451.102318216 + }, + "9.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 14807980.817563657, + "T_cal": 14807980.817563657 + }, + "9.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 640195.282145873, + "T_cal": 640195.282145873 + }, + "9.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 32147368.760505587, + "T_cal": 32147368.760505587 + }, + "9.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 31369271.903000325, + "T_cal": 31369271.903000325 + }, + "9.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 2343077.620179819, + "T_cal": 2343077.620179819 + }, + "10.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 26995528.23484038, + "T_cal": 26995528.23484038 + }, + "10.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 30623312.856559373, + "T_cal": 30623312.856559373 + }, + "10.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 14696503.59680074, + "T_cal": 14696503.59680074 + }, + "10.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 816463.8834357098, + "T_cal": 816463.8834357098 + }, + "10.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 32824751.711993486, + "T_cal": 32824751.711993486 + }, + "10.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 32988209.201422315, + "T_cal": 32988209.201422315 + }, + "10.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 2604280.4517786177, + "T_cal": 2604280.4517786177 + }, + "11.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 27885666.143974952, + "T_cal": 27885666.143974952 + }, + "11.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 29966915.981223837, + "T_cal": 29966915.981223837 + }, + "11.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 18354163.0248468, + "T_cal": 18354163.0248468 + }, + "11.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 774055.0968405588, + "T_cal": 774055.0968405588 + }, + "11.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 36298504.56562972, + "T_cal": 36298504.56562972 + }, + "11.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 37399445.937690735, + "T_cal": 37399445.937690735 + }, + "11.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 2946275.522105183, + "T_cal": 2946275.522105183 + }, + "12.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 37176687.66102626, + "T_cal": 37176687.66102626 + }, + "12.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39411258.50295681, + "T_cal": 39411258.50295681 + }, + "12.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 20373068.732661616, + "T_cal": 20373068.732661616 + }, + "12.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1026398.2289128569, + "T_cal": 1026398.2289128569 + }, + "12.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 38962720.388669126, + "T_cal": 38962720.388669126 + }, + "12.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 40763579.92419773, + "T_cal": 40763579.92419773 + }, + "12.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 3423418.0547610587, + "T_cal": 3423418.0547610587 + }, + "13.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39783861.868320554, + "T_cal": 39783861.868320554 + }, + "13.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 45018917.878322095, + "T_cal": 45018917.878322095 + }, + "13.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 26096478.08796837, + "T_cal": 26096478.08796837 + }, + "13.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1052517.2467130423, + "T_cal": 1052517.2467130423 + }, + "13.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 40982013.55999079, + "T_cal": 40982013.55999079 + }, + "13.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 44126156.415275246, + "T_cal": 44126156.415275246 + }, + "13.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 4184915.8135035345, + "T_cal": 4184915.8135035345 + }, + "14.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 37018227.33073595, + "T_cal": 37018227.33073595 + }, + "14.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39434127.285240084, + "T_cal": 39434127.285240084 + }, + "14.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 24977630.004054748, + "T_cal": 24977630.004054748 + }, + "14.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1321697.8823880195, + "T_cal": 1321697.8823880195 + }, + "14.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 44974626.91393885, + "T_cal": 44974626.91393885 + }, + "14.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 48546235.04470262, + "T_cal": 48546235.04470262 + }, + "14.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 4933667.934631426, + "T_cal": 4933667.934631426 + }, + "15.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39183390.05862564, + "T_cal": 39183390.05862564 + }, + "15.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 42428916.16894606, + "T_cal": 42428916.16894606 + }, + "15.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 27185844.25479764, + "T_cal": 27185844.25479764 + }, + "15.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1566934.9315340517, + "T_cal": 1566934.9315340517 + }, + "15.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 49157200.71268163, + "T_cal": 49157200.71268163 + }, + "15.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 53046712.07953124, + "T_cal": 53046712.07953124 + }, + "15.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 6198105.42750407, + "T_cal": 6198105.42750407 + }, + "16.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39726418.03975011, + "T_cal": 39726418.03975011 + }, + "16.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 42763441.79548803, + "T_cal": 42763441.79548803 + }, + "16.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 34081634.800825074, + "T_cal": 34081634.800825074 + }, + "16.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 2078580.3581403475, + "T_cal": 2078580.3581403475 + }, + "16.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 53526577.536419004, + "T_cal": 53526577.536419004 + }, + "16.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 56544512.05201172, + "T_cal": 56544512.05201172 + }, + "16.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 8068985.021892507, + "T_cal": 8068985.021892507 + }, + "17.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 49213815.36119798, + "T_cal": 49213815.36119798 + }, + "17.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 51094321.22252405, + "T_cal": 51094321.22252405 + }, + "17.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 37214008.645122886, + "T_cal": 37214008.645122886 + }, + "17.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1712786.0347415875, + "T_cal": 1712786.0347415875 + }, + "17.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 63587062.159486875, + "T_cal": 63587062.159486875 + }, + "17.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 64949533.02388008, + "T_cal": 64949533.02388008 + }, + "17.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 9018894.402179483, + "T_cal": 9018894.402179483 + }, + "18.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 60634287.77280918, + "T_cal": 60634287.77280918 + }, + "18.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 64831334.483361475, + "T_cal": 64831334.483361475 + }, + "18.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 48044737.34032382, + "T_cal": 48044737.34032382 + }, + "18.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1767358.542941892, + "T_cal": 1767358.542941892 + }, + "18.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 74723671.51761988, + "T_cal": 74723671.51761988 + }, + "18.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 73857104.66539228, + "T_cal": 73857104.66539228 + }, + "18.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 10711708.20267663, + "T_cal": 10711708.20267663 + }, + "19.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 55304963.76747595, + "T_cal": 55304963.76747595 + }, + "19.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 56528969.42952721, + "T_cal": 56528969.42952721 + }, + "19.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 46749382.679476425, + "T_cal": 46749382.679476425 + }, + "19.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1909976.1048511192, + "T_cal": 1909976.1048511192 + }, + "19.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 83881343.36797214, + "T_cal": 83881343.36797214 + }, + "19.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 81299879.09767886, + "T_cal": 81299879.09767886 + }, + "19.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 12105003.388167828, + "T_cal": 12105003.388167828 + }, + "20.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 55406892.56403369, + "T_cal": 55406892.56403369 + }, + "20.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 57890868.59019267, + "T_cal": 57890868.59019267 + }, + "20.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 51750677.14486361, + "T_cal": 51750677.14486361 + }, + "20.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 2223282.921577907, + "T_cal": 2223282.921577907 + }, + "20.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 88847283.2477088, + "T_cal": 88847283.2477088 + }, + "20.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 84743306.78932326, + "T_cal": 84743306.78932326 + }, + "20.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 14629944.451741323, + "T_cal": 14629944.451741323 + }, + "21.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 63629217.53905773, + "T_cal": 63629217.53905773 + }, + "21.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 65724236.697300464, + "T_cal": 65724236.697300464 + }, + "21.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 60855097.43148553, + "T_cal": 60855097.43148553 + }, + "21.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 2177357.1019635256, + "T_cal": 2177357.1019635256 + }, + "21.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 100157646.62212129, + "T_cal": 100157646.62212129 + }, + "21.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 93378934.56804733, + "T_cal": 93378934.56804733 + }, + "21.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 16246214.349225592, + "T_cal": 16246214.349225592 + }, + "22.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 73927201.35627109, + "T_cal": 73927201.35627109 + }, + "22.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 77179331.82880369, + "T_cal": 77179331.82880369 + }, + "22.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 67726849.51546389, + "T_cal": 67726849.51546389 + }, + "22.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 3074665.35284698, + "T_cal": 3074665.35284698 + }, + "22.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 111125496.81619574, + "T_cal": 111125496.81619574 + }, + "22.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 101993949.9344372, + "T_cal": 101993949.9344372 + }, + "22.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 18971098.848933563, + "T_cal": 18971098.848933563 + }, + "23.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 97634615.48427142, + "T_cal": 97634615.48427142 + }, + "23.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 98922618.71968251, + "T_cal": 98922618.71968251 + }, + "23.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 86658830.16723555, + "T_cal": 86658830.16723555 + }, + "23.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 3519204.682988597, + "T_cal": 3519204.682988597 + }, + "23.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 125824435.47619548, + "T_cal": 125824435.47619548 + }, + "23.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 115975936.4452641, + "T_cal": 115975936.4452641 + }, + "23.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 20835886.751074, + "T_cal": 20835886.751074 + }, + "24.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 73169262.10265872, + "T_cal": 73169262.10265872 + }, + "24.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 75418927.0367334, + "T_cal": 75418927.0367334 + }, + "24.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 82690646.43827185, + "T_cal": 82690646.43827185 + }, + "24.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 3687126.166784588, + "T_cal": 3687126.166784588 + }, + "24.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 139489695.71145046, + "T_cal": 139489695.71145046 + }, + "24.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 128657833.76944333, + "T_cal": 128657833.76944333 + }, + "24.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 22713822.37057341, + "T_cal": 22713822.37057341 + }, + "25.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 108939374.45137435, + "T_cal": 108939374.45137435 + }, + "25.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 113346515.38954216, + "T_cal": 113346515.38954216 + }, + "25.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 111969270.95768926, + "T_cal": 111969270.95768926 + }, + "25.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 3369497.6247471585, + "T_cal": 3369497.6247471585 + }, + "25.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 153477232.42426157, + "T_cal": 153477232.42426157 + }, + "25.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 141666077.36553234, + "T_cal": 141666077.36553234 + }, + "25.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 24071119.081784677, + "T_cal": 24071119.081784677 + }, + "26.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 88902098.68380079, + "T_cal": 88902098.68380079 + }, + "26.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 93676824.95389336, + "T_cal": 93676824.95389336 + }, + "26.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 106553205.94253726, + "T_cal": 106553205.94253726 + }, + "26.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 4257682.498286205, + "T_cal": 4257682.498286205 + }, + "26.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 168820570.65894625, + "T_cal": 168820570.65894625 + }, + "26.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 156327167.4446779, + "T_cal": 156327167.4446779 + }, + "26.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 26913816.820288904, + "T_cal": 26913816.820288904 + }, + "27.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 126298426.07064164, + "T_cal": 126298426.07064164 + }, + "27.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 132880719.95263419, + "T_cal": 132880719.95263419 + }, + "27.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 122812339.89492148, + "T_cal": 122812339.89492148 + }, + "27.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 5298282.005891386, + "T_cal": 5298282.005891386 + }, + "27.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 186024394.6277617, + "T_cal": 186024394.6277617 + }, + "27.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 172826957.01387775, + "T_cal": 172826957.01387775 + }, + "27.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 30745264.468438882, + "T_cal": 30745264.468438882 + }, + "28.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 115041668.36365198, + "T_cal": 115041668.36365198 + }, + "28.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 123108381.5654003, + "T_cal": 123108381.5654003 + }, + "28.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 134251802.03076518, + "T_cal": 134251802.03076518 + }, + "28.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 5424623.196999095, + "T_cal": 5424623.196999095 + }, + "28.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 192492884.72225767, + "T_cal": 192492884.72225767 + }, + "28.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 183214165.90316838, + "T_cal": 183214165.90316838 + }, + "28.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 36163704.44775533, + "T_cal": 36163704.44775533 + }, + "29.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 88150586.37762496, + "T_cal": 88150586.37762496 + }, + "29.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 93005789.70568019, + "T_cal": 93005789.70568019 + }, + "29.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 123287808.43405941, + "T_cal": 123287808.43405941 + }, + "29.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 6492358.462332103, + "T_cal": 6492358.462332103 + }, + "29.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 198965340.1598056, + "T_cal": 198965340.1598056 + }, + "29.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 189928121.6632526, + "T_cal": 189928121.6632526 + }, + "29.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 41748425.68348543, + "T_cal": 41748425.68348543 + }, + "30.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 108400125.89485452, + "T_cal": 108400125.89485452 + }, + "30.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 114947126.58425388, + "T_cal": 114947126.58425388 + }, + "30.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 141309618.88119125, + "T_cal": 141309618.88119125 + }, + "30.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 6535215.355932907, + "T_cal": 6535215.355932907 + }, + "30.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 198918618.52056333, + "T_cal": 198918618.52056333 + }, + "30.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 184481538.96409804, + "T_cal": 184481538.96409804 + }, + "30.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 48089070.14869043, + "T_cal": 48089070.14869043 + }, + "31.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 55561283.22422707, + "T_cal": 55561283.22422707 + }, + "31.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 62158039.85492205, + "T_cal": 62158039.85492205 + }, + "31.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 71547520.77123587, + "T_cal": 71547520.77123587 + }, + "31.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 5738000.461189147, + "T_cal": 5738000.461189147 + }, + "31.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 157805009.62711662, + "T_cal": 157805009.62711662 + }, + "31.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 142532753.2012003, + "T_cal": 142532753.2012003 + }, + "31.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 53183822.605515584, + "T_cal": 53183822.605515584 + } + }, + "calib_spec": "wikitext2:256", + "nsamples": 256, + "seed": 42, + "n_calib_sequences": 255 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..dc592da --- /dev/null +++ b/config.json @@ -0,0 +1,30 @@ +{ + "_name_or_path": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 11008, + "max_position_embeddings": 4096, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 32, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.46.3", + "use_cache": true, + "vocab_size": 32000 +} diff --git a/evaluation/safety/overrefusal.json b/evaluation/safety/overrefusal.json new file mode 100644 index 0000000..703f71f --- /dev/null +++ b/evaluation/safety/overrefusal.json @@ -0,0 +1,284 @@ +{ + "tag": "l2_base_k0_a1p0_free_remove30", + "model": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove30/checkpoint", + "judge_model": "/lustre/jeesup0103/svd-safety/assets/models/Wildguard", + "judge": "WildGuard response-refusal classifier", + "protocol": "chat_template_greedy_then_wildguard", + "max_new_tokens": 512, + "limit_per_benchmark": null, + "datasets": { + "xstest_safe": { + "path": "/home/jeesup0103/svd_safety/data/xstest_safe.jsonl", + "n": 250, + "n_judge_parsed": 250, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 250, + "scored_fraction": 1.0, + "n_refused": 41, + "refusal_rate": 0.164, + "reliable": true, + "by_category": { + "definitions": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 0, + "refusal_rate": 0.0, + "reliable": true + }, + "figurative_language": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 3, + "refusal_rate": 0.12, + "reliable": true + }, + "historical_events": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 0, + "refusal_rate": 0.0, + "reliable": true + }, + "homonyms": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 4, + "refusal_rate": 0.16, + "reliable": true + }, + "nons_group_real_discr": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 3, + "refusal_rate": 0.12, + "reliable": true + }, + "privacy_fictional": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 11, + "refusal_rate": 0.44, + "reliable": true + }, + "privacy_public": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 2, + "refusal_rate": 0.08, + "reliable": true + }, + "real_group_nons_discr": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 3, + "refusal_rate": 0.12, + "reliable": true + }, + "safe_contexts": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 8, + "refusal_rate": 0.32, + "reliable": true + }, + "safe_targets": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 7, + "refusal_rate": 0.28, + "reliable": true + } + } + }, + "orbench_hard_1k": { + "path": "/home/jeesup0103/svd_safety/data/or_bench_hard_1k.csv", + "n": 1319, + "n_judge_parsed": 1319, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 1, + "n_scored": 1318, + "scored_fraction": 0.9992, + "n_refused": 540, + "refusal_rate": 0.40971168437025796, + "reliable": true, + "by_category": { + "deception": { + "n": 72, + "n_judge_parsed": 72, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 72, + "scored_fraction": 1.0, + "n_refused": 26, + "refusal_rate": 0.3611111111111111, + "reliable": true + }, + "harassment": { + "n": 41, + "n_judge_parsed": 41, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 41, + "scored_fraction": 1.0, + "n_refused": 22, + "refusal_rate": 0.5365853658536586, + "reliable": true + }, + "harmful": { + "n": 106, + "n_judge_parsed": 106, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 106, + "scored_fraction": 1.0, + "n_refused": 38, + "refusal_rate": 0.3584905660377358, + "reliable": true + }, + "hate": { + "n": 54, + "n_judge_parsed": 54, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 54, + "scored_fraction": 1.0, + "n_refused": 27, + "refusal_rate": 0.5, + "reliable": true + }, + "illegal": { + "n": 527, + "n_judge_parsed": 527, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 1, + "n_scored": 526, + "scored_fraction": 0.9981, + "n_refused": 220, + "refusal_rate": 0.41825095057034223, + "reliable": true + }, + "privacy": { + "n": 199, + "n_judge_parsed": 199, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 199, + "scored_fraction": 1.0, + "n_refused": 86, + "refusal_rate": 0.4321608040201005, + "reliable": true + }, + "self-harm": { + "n": 63, + "n_judge_parsed": 63, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 63, + "scored_fraction": 1.0, + "n_refused": 20, + "refusal_rate": 0.31746031746031744, + "reliable": true + }, + "sexual": { + "n": 66, + "n_judge_parsed": 66, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 66, + "scored_fraction": 1.0, + "n_refused": 11, + "refusal_rate": 0.16666666666666666, + "reliable": true + }, + "unethical": { + "n": 125, + "n_judge_parsed": 125, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 125, + "scored_fraction": 1.0, + "n_refused": 63, + "refusal_rate": 0.504, + "reliable": true + }, + "violence": { + "n": 66, + "n_judge_parsed": 66, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 66, + "scored_fraction": 1.0, + "n_refused": 27, + "refusal_rate": 0.4090909090909091, + "reliable": true + } + } + } + }, + "macro_over_refusal_rate": 0.286855842185129, + "reliable": true, + "note": "Refusal rates use WildGuard labels over informative responses. Read reliable/scored_fraction first: compressed models that emit uninterpretable output do not receive an artificially low over-refusal score. Macro rate weights the two benchmarks equally." +} diff --git a/evaluation/safety/strongreject_summary.json b/evaluation/safety/strongreject_summary.json new file mode 100644 index 0000000..ce4d107 --- /dev/null +++ b/evaluation/safety/strongreject_summary.json @@ -0,0 +1,13 @@ +{ + "tag": "l2_base_k0_a1p0_free_remove30", + "judge_model": "/lustre/jeesup0103/svd-safety/assets/models/HarmBench-Llama-2-13b-cls", + "protocol": "strongreject_direct_chat_greedy", + "n_prompts": 313, + "n_harmful": 16, + "n_not_harmful": 297, + "attack_success_rate": 0.051118210862619806, + "mean_p_yes": 0.0502080546693573, + "argmax_offlist_rate": 0.0, + "per_prompt_jsonl": "/lustre/jeesup0103/svd-safety/results/resmix/evaluation_ft/l2_base_k0_a1p0_free_remove30/safety/strongreject_per_prompt.jsonl", + "per_prompt_csv": "/lustre/jeesup0103/svd-safety/results/resmix/evaluation_ft/l2_base_k0_a1p0_free_remove30/safety/strongreject_per_prompt.csv" +} diff --git a/evaluation/safety/summary.json b/evaluation/safety/summary.json new file mode 100644 index 0000000..c9f9646 --- /dev/null +++ b/evaluation/safety/summary.json @@ -0,0 +1,13 @@ +{ + "tag": "l2_base_k0_a1p0_free_remove30", + "judge_model": "/lustre/jeesup0103/svd-safety/assets/models/HarmBench-Llama-2-13b-cls", + "protocol": "advbench_direct_chat_greedy", + "n_prompts": 520, + "n_harmful": 33, + "n_not_harmful": 487, + "attack_success_rate": 0.06346153846153846, + "mean_p_yes": 0.0640963200154443, + "argmax_offlist_rate": 0.0, + "per_prompt_jsonl": "/lustre/jeesup0103/svd-safety/results/resmix/evaluation_ft/l2_base_k0_a1p0_free_remove30/safety/per_prompt.jsonl", + "per_prompt_csv": "/lustre/jeesup0103/svd-safety/results/resmix/evaluation_ft/l2_base_k0_a1p0_free_remove30/safety/per_prompt.csv" +} diff --git a/evaluation/utility/ppl.json b/evaluation/utility/ppl.json new file mode 100644 index 0000000..3cbdb19 --- /dev/null +++ b/evaluation/utility/ppl.json @@ -0,0 +1,6 @@ +{ + "tag": "ppl", + "model": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove30/checkpoint", + "wikitext2_ppl": 9.784844398498535, + "n_tokens": 341302 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..aaaeef2 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,10 @@ +{ + "bos_token_id": 1, + "do_sample": true, + "eos_token_id": 2, + "max_length": 4096, + "pad_token_id": 0, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.46.3" +} diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000..b0fb24b --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c64ab59f88ea5f9a5f452ea6be8d32dae6c761490604466927bcf36d78729b9 +size 4938985352 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000..dbf7271 --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4d731ede9401ba1511a174cbd026f8a224ceb99d122015164abbaa1fa418981 +size 4947390880 diff --git a/model-00003-of-00003.safetensors b/model-00003-of-00003.safetensors new file mode 100644 index 0000000..62a908c --- /dev/null +++ b/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:758ecea9cff9efa20590201bf72bf42e0ebd29192d786227e5ff2cedfdf32af4 +size 3590488816 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..13674e5 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 13476831232 + }, + "weight_map": { + "lm_head.weight": "model-00003-of-00003.safetensors", + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/resmix/compression.json b/resmix/compression.json new file mode 100644 index 0000000..0619182 --- /dev/null +++ b/resmix/compression.json @@ -0,0 +1,1850 @@ +{ + "source": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", + "upstream": "AIoT-MLSys-Lab/SVD-LLM", + "step": "whiten", + "ratio_removed": 0.3, + "internal_ratio_kept": 0.7, + "nsamples": 256, + "seed": 42, + "calib_spec": "wikitext2:256", + "calib": [ + { + "name": "wikitext2", + "n": 255, + "source": "upstream get_calib_train_data" + } + ], + "n_calib_sequences": 255, + "rank_by": "resmix", + "whitening": { + "lam": 0.0, + "norm": "token", + "ridge": 0.0, + "token_mask": "nospecial", + "utility_cache": "/lustre/jeesup0103/svd-safety/work/whiten_cache/utility_wikitext256", + "utility_calib": "wikitext2:256", + "utility_tokens": 522240 + }, + "whiten_lam": 0.0, + "lam": 0.0, + "fisher_scores": null, + "fisher_num": null, + "fisher_den": null, + "fisher_form": null, + "fisher_control": null, + "fisher_floor": null, + "safety_calib": null, + "resmix_k": 0.0, + "resmix_k_abs": null, + "resmix_alpha": 1.0, + "n_safety_tokens": null, + "ranking": { + "n_matrices": 224, + "alpha": 1.0, + "k_s_total": 0, + "k_s_frac_of_rank": 0.0, + "utility_error_ratio": 1.0, + "resmix_k": 0.0, + "resmix_k_abs": null, + "resmix_alpha": 1.0 + }, + "ranking_per_matrix": { + "0.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39.388282935811475, + "T_cal": 39.388282935811475 + }, + "0.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 34.64168906039899, + "T_cal": 34.64168906039899 + }, + "0.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 2180.4545400725456, + "T_cal": 2180.4545400725456 + }, + "0.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 3.236008245712793, + "T_cal": 3.236008245712793 + }, + "0.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 233290.09868813131, + "T_cal": 233290.09868813131 + }, + "0.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 239977.84535169817, + "T_cal": 239977.84535169817 + }, + "0.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 4897.100459002034, + "T_cal": 4897.100459002034 + }, + "1.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 28729.716566027415, + "T_cal": 28729.716566027415 + }, + "1.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 21643.320093676906, + "T_cal": 21643.320093676906 + }, + "1.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 51962.35635288252, + "T_cal": 51962.35635288252 + }, + "1.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 467.6564444913811, + "T_cal": 467.6564444913811 + }, + "1.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 902488.3564397852, + "T_cal": 902488.3564397852 + }, + "1.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 1023626.5545441816, + "T_cal": 1023626.5545441816 + }, + "1.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 59103.256723517115, + "T_cal": 59103.256723517115 + }, + "2.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1774134.0302654633, + "T_cal": 1774134.0302654633 + }, + "2.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1778735.8435498467, + "T_cal": 1778735.8435498467 + }, + "2.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1784142.2006091198, + "T_cal": 1784142.2006091198 + }, + "2.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 24332.456126517485, + "T_cal": 24332.456126517485 + }, + "2.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 3897275.9891598956, + "T_cal": 3897275.9891598956 + }, + "2.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 3902478.238080211, + "T_cal": 3902478.238080211 + }, + "2.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 168321.99261231394, + "T_cal": 168321.99261231394 + }, + "3.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 12643629.215040559, + "T_cal": 12643629.215040559 + }, + "3.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 13716496.810724786, + "T_cal": 13716496.810724786 + }, + "3.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 7078173.434168732, + "T_cal": 7078173.434168732 + }, + "3.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39460.50467348984, + "T_cal": 39460.50467348984 + }, + "3.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 10688950.318662636, + "T_cal": 10688950.318662636 + }, + "3.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 9919457.723038245, + "T_cal": 9919457.723038245 + }, + "3.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 297727.46114792966, + "T_cal": 297727.46114792966 + }, + "4.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 12380687.063937424, + "T_cal": 12380687.063937424 + }, + "4.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 13230745.349890629, + "T_cal": 13230745.349890629 + }, + "4.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 7377412.447694868, + "T_cal": 7377412.447694868 + }, + "4.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 88498.14542927106, + "T_cal": 88498.14542927106 + }, + "4.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 15917695.579375152, + "T_cal": 15917695.579375152 + }, + "4.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 14251862.833006734, + "T_cal": 14251862.833006734 + }, + "4.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 548192.1997935446, + "T_cal": 548192.1997935446 + }, + "5.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 15545430.394920252, + "T_cal": 15545430.394920252 + }, + "5.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 16787557.251722403, + "T_cal": 16787557.251722403 + }, + "5.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 9192295.965431035, + "T_cal": 9192295.965431035 + }, + "5.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 116922.83781016959, + "T_cal": 116922.83781016959 + }, + "5.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 19895489.005969524, + "T_cal": 19895489.005969524 + }, + "5.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 17966035.376588047, + "T_cal": 17966035.376588047 + }, + "5.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 833079.9087956194, + "T_cal": 833079.9087956194 + }, + "6.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 21172200.32657173, + "T_cal": 21172200.32657173 + }, + "6.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 23004804.61069621, + "T_cal": 23004804.61069621 + }, + "6.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 12149843.699887238, + "T_cal": 12149843.699887238 + }, + "6.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 203828.19718049304, + "T_cal": 203828.19718049304 + }, + "6.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 25536093.872205686, + "T_cal": 25536093.872205686 + }, + "6.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 22666967.386803932, + "T_cal": 22666967.386803932 + }, + "6.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 1194725.368004628, + "T_cal": 1194725.368004628 + }, + "7.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 23996831.82645592, + "T_cal": 23996831.82645592 + }, + "7.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 25165174.584726967, + "T_cal": 25165174.584726967 + }, + "7.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 13360452.246222524, + "T_cal": 13360452.246222524 + }, + "7.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 286590.00475342566, + "T_cal": 286590.00475342566 + }, + "7.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 29828875.85005765, + "T_cal": 29828875.85005765 + }, + "7.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 26955563.39740563, + "T_cal": 26955563.39740563 + }, + "7.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 1598230.184621131, + "T_cal": 1598230.184621131 + }, + "8.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 25595703.640618827, + "T_cal": 25595703.640618827 + }, + "8.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 27773611.036283582, + "T_cal": 27773611.036283582 + }, + "8.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 13208919.58704463, + "T_cal": 13208919.58704463 + }, + "8.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 429615.15508663515, + "T_cal": 429615.15508663515 + }, + "8.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 30698682.080411024, + "T_cal": 30698682.080411024 + }, + "8.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 29097623.450163342, + "T_cal": 29097623.450163342 + }, + "8.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 1943908.4716851546, + "T_cal": 1943908.4716851546 + }, + "9.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 30298298.18147658, + "T_cal": 30298298.18147658 + }, + "9.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 32860451.102318216, + "T_cal": 32860451.102318216 + }, + "9.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 14807980.817563657, + "T_cal": 14807980.817563657 + }, + "9.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 640195.282145873, + "T_cal": 640195.282145873 + }, + "9.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 32147368.760505587, + "T_cal": 32147368.760505587 + }, + "9.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 31369271.903000325, + "T_cal": 31369271.903000325 + }, + "9.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 2343077.620179819, + "T_cal": 2343077.620179819 + }, + "10.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 26995528.23484038, + "T_cal": 26995528.23484038 + }, + "10.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 30623312.856559373, + "T_cal": 30623312.856559373 + }, + "10.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 14696503.59680074, + "T_cal": 14696503.59680074 + }, + "10.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 816463.8834357098, + "T_cal": 816463.8834357098 + }, + "10.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 32824751.711993486, + "T_cal": 32824751.711993486 + }, + "10.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 32988209.201422315, + "T_cal": 32988209.201422315 + }, + "10.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 2604280.4517786177, + "T_cal": 2604280.4517786177 + }, + "11.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 27885666.143974952, + "T_cal": 27885666.143974952 + }, + "11.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 29966915.981223837, + "T_cal": 29966915.981223837 + }, + "11.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 18354163.0248468, + "T_cal": 18354163.0248468 + }, + "11.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 774055.0968405588, + "T_cal": 774055.0968405588 + }, + "11.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 36298504.56562972, + "T_cal": 36298504.56562972 + }, + "11.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 37399445.937690735, + "T_cal": 37399445.937690735 + }, + "11.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 2946275.522105183, + "T_cal": 2946275.522105183 + }, + "12.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 37176687.66102626, + "T_cal": 37176687.66102626 + }, + "12.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39411258.50295681, + "T_cal": 39411258.50295681 + }, + "12.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 20373068.732661616, + "T_cal": 20373068.732661616 + }, + "12.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1026398.2289128569, + "T_cal": 1026398.2289128569 + }, + "12.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 38962720.388669126, + "T_cal": 38962720.388669126 + }, + "12.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 40763579.92419773, + "T_cal": 40763579.92419773 + }, + "12.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 3423418.0547610587, + "T_cal": 3423418.0547610587 + }, + "13.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39783861.868320554, + "T_cal": 39783861.868320554 + }, + "13.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 45018917.878322095, + "T_cal": 45018917.878322095 + }, + "13.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 26096478.08796837, + "T_cal": 26096478.08796837 + }, + "13.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1052517.2467130423, + "T_cal": 1052517.2467130423 + }, + "13.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 40982013.55999079, + "T_cal": 40982013.55999079 + }, + "13.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 44126156.415275246, + "T_cal": 44126156.415275246 + }, + "13.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 4184915.8135035345, + "T_cal": 4184915.8135035345 + }, + "14.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 37018227.33073595, + "T_cal": 37018227.33073595 + }, + "14.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39434127.285240084, + "T_cal": 39434127.285240084 + }, + "14.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 24977630.004054748, + "T_cal": 24977630.004054748 + }, + "14.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1321697.8823880195, + "T_cal": 1321697.8823880195 + }, + "14.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 44974626.91393885, + "T_cal": 44974626.91393885 + }, + "14.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 48546235.04470262, + "T_cal": 48546235.04470262 + }, + "14.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 4933667.934631426, + "T_cal": 4933667.934631426 + }, + "15.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39183390.05862564, + "T_cal": 39183390.05862564 + }, + "15.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 42428916.16894606, + "T_cal": 42428916.16894606 + }, + "15.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 27185844.25479764, + "T_cal": 27185844.25479764 + }, + "15.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1566934.9315340517, + "T_cal": 1566934.9315340517 + }, + "15.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 49157200.71268163, + "T_cal": 49157200.71268163 + }, + "15.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 53046712.07953124, + "T_cal": 53046712.07953124 + }, + "15.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 6198105.42750407, + "T_cal": 6198105.42750407 + }, + "16.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 39726418.03975011, + "T_cal": 39726418.03975011 + }, + "16.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 42763441.79548803, + "T_cal": 42763441.79548803 + }, + "16.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 34081634.800825074, + "T_cal": 34081634.800825074 + }, + "16.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 2078580.3581403475, + "T_cal": 2078580.3581403475 + }, + "16.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 53526577.536419004, + "T_cal": 53526577.536419004 + }, + "16.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 56544512.05201172, + "T_cal": 56544512.05201172 + }, + "16.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 8068985.021892507, + "T_cal": 8068985.021892507 + }, + "17.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 49213815.36119798, + "T_cal": 49213815.36119798 + }, + "17.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 51094321.22252405, + "T_cal": 51094321.22252405 + }, + "17.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 37214008.645122886, + "T_cal": 37214008.645122886 + }, + "17.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1712786.0347415875, + "T_cal": 1712786.0347415875 + }, + "17.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 63587062.159486875, + "T_cal": 63587062.159486875 + }, + "17.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 64949533.02388008, + "T_cal": 64949533.02388008 + }, + "17.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 9018894.402179483, + "T_cal": 9018894.402179483 + }, + "18.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 60634287.77280918, + "T_cal": 60634287.77280918 + }, + "18.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 64831334.483361475, + "T_cal": 64831334.483361475 + }, + "18.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 48044737.34032382, + "T_cal": 48044737.34032382 + }, + "18.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1767358.542941892, + "T_cal": 1767358.542941892 + }, + "18.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 74723671.51761988, + "T_cal": 74723671.51761988 + }, + "18.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 73857104.66539228, + "T_cal": 73857104.66539228 + }, + "18.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 10711708.20267663, + "T_cal": 10711708.20267663 + }, + "19.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 55304963.76747595, + "T_cal": 55304963.76747595 + }, + "19.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 56528969.42952721, + "T_cal": 56528969.42952721 + }, + "19.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 46749382.679476425, + "T_cal": 46749382.679476425 + }, + "19.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 1909976.1048511192, + "T_cal": 1909976.1048511192 + }, + "19.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 83881343.36797214, + "T_cal": 83881343.36797214 + }, + "19.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 81299879.09767886, + "T_cal": 81299879.09767886 + }, + "19.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 12105003.388167828, + "T_cal": 12105003.388167828 + }, + "20.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 55406892.56403369, + "T_cal": 55406892.56403369 + }, + "20.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 57890868.59019267, + "T_cal": 57890868.59019267 + }, + "20.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 51750677.14486361, + "T_cal": 51750677.14486361 + }, + "20.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 2223282.921577907, + "T_cal": 2223282.921577907 + }, + "20.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 88847283.2477088, + "T_cal": 88847283.2477088 + }, + "20.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 84743306.78932326, + "T_cal": 84743306.78932326 + }, + "20.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 14629944.451741323, + "T_cal": 14629944.451741323 + }, + "21.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 63629217.53905773, + "T_cal": 63629217.53905773 + }, + "21.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 65724236.697300464, + "T_cal": 65724236.697300464 + }, + "21.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 60855097.43148553, + "T_cal": 60855097.43148553 + }, + "21.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 2177357.1019635256, + "T_cal": 2177357.1019635256 + }, + "21.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 100157646.62212129, + "T_cal": 100157646.62212129 + }, + "21.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 93378934.56804733, + "T_cal": 93378934.56804733 + }, + "21.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 16246214.349225592, + "T_cal": 16246214.349225592 + }, + "22.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 73927201.35627109, + "T_cal": 73927201.35627109 + }, + "22.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 77179331.82880369, + "T_cal": 77179331.82880369 + }, + "22.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 67726849.51546389, + "T_cal": 67726849.51546389 + }, + "22.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 3074665.35284698, + "T_cal": 3074665.35284698 + }, + "22.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 111125496.81619574, + "T_cal": 111125496.81619574 + }, + "22.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 101993949.9344372, + "T_cal": 101993949.9344372 + }, + "22.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 18971098.848933563, + "T_cal": 18971098.848933563 + }, + "23.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 97634615.48427142, + "T_cal": 97634615.48427142 + }, + "23.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 98922618.71968251, + "T_cal": 98922618.71968251 + }, + "23.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 86658830.16723555, + "T_cal": 86658830.16723555 + }, + "23.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 3519204.682988597, + "T_cal": 3519204.682988597 + }, + "23.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 125824435.47619548, + "T_cal": 125824435.47619548 + }, + "23.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 115975936.4452641, + "T_cal": 115975936.4452641 + }, + "23.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 20835886.751074, + "T_cal": 20835886.751074 + }, + "24.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 73169262.10265872, + "T_cal": 73169262.10265872 + }, + "24.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 75418927.0367334, + "T_cal": 75418927.0367334 + }, + "24.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 82690646.43827185, + "T_cal": 82690646.43827185 + }, + "24.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 3687126.166784588, + "T_cal": 3687126.166784588 + }, + "24.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 139489695.71145046, + "T_cal": 139489695.71145046 + }, + "24.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 128657833.76944333, + "T_cal": 128657833.76944333 + }, + "24.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 22713822.37057341, + "T_cal": 22713822.37057341 + }, + "25.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 108939374.45137435, + "T_cal": 108939374.45137435 + }, + "25.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 113346515.38954216, + "T_cal": 113346515.38954216 + }, + "25.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 111969270.95768926, + "T_cal": 111969270.95768926 + }, + "25.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 3369497.6247471585, + "T_cal": 3369497.6247471585 + }, + "25.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 153477232.42426157, + "T_cal": 153477232.42426157 + }, + "25.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 141666077.36553234, + "T_cal": 141666077.36553234 + }, + "25.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 24071119.081784677, + "T_cal": 24071119.081784677 + }, + "26.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 88902098.68380079, + "T_cal": 88902098.68380079 + }, + "26.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 93676824.95389336, + "T_cal": 93676824.95389336 + }, + "26.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 106553205.94253726, + "T_cal": 106553205.94253726 + }, + "26.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 4257682.498286205, + "T_cal": 4257682.498286205 + }, + "26.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 168820570.65894625, + "T_cal": 168820570.65894625 + }, + "26.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 156327167.4446779, + "T_cal": 156327167.4446779 + }, + "26.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 26913816.820288904, + "T_cal": 26913816.820288904 + }, + "27.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 126298426.07064164, + "T_cal": 126298426.07064164 + }, + "27.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 132880719.95263419, + "T_cal": 132880719.95263419 + }, + "27.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 122812339.89492148, + "T_cal": 122812339.89492148 + }, + "27.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 5298282.005891386, + "T_cal": 5298282.005891386 + }, + "27.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 186024394.6277617, + "T_cal": 186024394.6277617 + }, + "27.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 172826957.01387775, + "T_cal": 172826957.01387775 + }, + "27.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 30745264.468438882, + "T_cal": 30745264.468438882 + }, + "28.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 115041668.36365198, + "T_cal": 115041668.36365198 + }, + "28.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 123108381.5654003, + "T_cal": 123108381.5654003 + }, + "28.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 134251802.03076518, + "T_cal": 134251802.03076518 + }, + "28.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 5424623.196999095, + "T_cal": 5424623.196999095 + }, + "28.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 192492884.72225767, + "T_cal": 192492884.72225767 + }, + "28.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 183214165.90316838, + "T_cal": 183214165.90316838 + }, + "28.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 36163704.44775533, + "T_cal": 36163704.44775533 + }, + "29.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 88150586.37762496, + "T_cal": 88150586.37762496 + }, + "29.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 93005789.70568019, + "T_cal": 93005789.70568019 + }, + "29.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 123287808.43405941, + "T_cal": 123287808.43405941 + }, + "29.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 6492358.462332103, + "T_cal": 6492358.462332103 + }, + "29.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 198965340.1598056, + "T_cal": 198965340.1598056 + }, + "29.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 189928121.6632526, + "T_cal": 189928121.6632526 + }, + "29.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 41748425.68348543, + "T_cal": 41748425.68348543 + }, + "30.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 108400125.89485452, + "T_cal": 108400125.89485452 + }, + "30.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 114947126.58425388, + "T_cal": 114947126.58425388 + }, + "30.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 141309618.88119125, + "T_cal": 141309618.88119125 + }, + "30.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 6535215.355932907, + "T_cal": 6535215.355932907 + }, + "30.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 198918618.52056333, + "T_cal": 198918618.52056333 + }, + "30.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 184481538.96409804, + "T_cal": 184481538.96409804 + }, + "30.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 48089070.14869043, + "T_cal": 48089070.14869043 + }, + "31.q_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 55561283.22422707, + "T_cal": 55561283.22422707 + }, + "31.k_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 62158039.85492205, + "T_cal": 62158039.85492205 + }, + "31.v_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 71547520.77123587, + "T_cal": 71547520.77123587 + }, + "31.o_proj": { + "r": 1433, + "k_u": 1433, + "k_s": 0, + "T_cal_prefix": 5738000.461189147, + "T_cal": 5738000.461189147 + }, + "31.gate_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 157805009.62711662, + "T_cal": 157805009.62711662 + }, + "31.up_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 142532753.2012003, + "T_cal": 142532753.2012003 + }, + "31.down_proj": { + "r": 2089, + "k_u": 2089, + "k_s": 0, + "T_cal_prefix": 53183822.605515584, + "T_cal": 53183822.605515584 + } + }, + "ranks": { + "0.q_proj": 1433, + "0.k_proj": 1433, + "0.v_proj": 1433, + "0.o_proj": 1433, + "0.gate_proj": 2089, + "0.up_proj": 2089, + "0.down_proj": 2089, + "1.q_proj": 1433, + "1.k_proj": 1433, + "1.v_proj": 1433, + "1.o_proj": 1433, + "1.gate_proj": 2089, + "1.up_proj": 2089, + "1.down_proj": 2089, + "2.q_proj": 1433, + "2.k_proj": 1433, + "2.v_proj": 1433, + "2.o_proj": 1433, + "2.gate_proj": 2089, + "2.up_proj": 2089, + "2.down_proj": 2089, + "3.q_proj": 1433, + "3.k_proj": 1433, + "3.v_proj": 1433, + "3.o_proj": 1433, + "3.gate_proj": 2089, + "3.up_proj": 2089, + "3.down_proj": 2089, + "4.q_proj": 1433, + "4.k_proj": 1433, + "4.v_proj": 1433, + "4.o_proj": 1433, + "4.gate_proj": 2089, + "4.up_proj": 2089, + "4.down_proj": 2089, + "5.q_proj": 1433, + "5.k_proj": 1433, + "5.v_proj": 1433, + "5.o_proj": 1433, + "5.gate_proj": 2089, + "5.up_proj": 2089, + "5.down_proj": 2089, + "6.q_proj": 1433, + "6.k_proj": 1433, + "6.v_proj": 1433, + "6.o_proj": 1433, + "6.gate_proj": 2089, + "6.up_proj": 2089, + "6.down_proj": 2089, + "7.q_proj": 1433, + "7.k_proj": 1433, + "7.v_proj": 1433, + "7.o_proj": 1433, + "7.gate_proj": 2089, + "7.up_proj": 2089, + "7.down_proj": 2089, + "8.q_proj": 1433, + "8.k_proj": 1433, + "8.v_proj": 1433, + "8.o_proj": 1433, + "8.gate_proj": 2089, + "8.up_proj": 2089, + "8.down_proj": 2089, + "9.q_proj": 1433, + "9.k_proj": 1433, + "9.v_proj": 1433, + "9.o_proj": 1433, + "9.gate_proj": 2089, + "9.up_proj": 2089, + "9.down_proj": 2089, + "10.q_proj": 1433, + "10.k_proj": 1433, + "10.v_proj": 1433, + "10.o_proj": 1433, + "10.gate_proj": 2089, + "10.up_proj": 2089, + "10.down_proj": 2089, + "11.q_proj": 1433, + "11.k_proj": 1433, + "11.v_proj": 1433, + "11.o_proj": 1433, + "11.gate_proj": 2089, + "11.up_proj": 2089, + "11.down_proj": 2089, + "12.q_proj": 1433, + "12.k_proj": 1433, + "12.v_proj": 1433, + "12.o_proj": 1433, + "12.gate_proj": 2089, + "12.up_proj": 2089, + "12.down_proj": 2089, + "13.q_proj": 1433, + "13.k_proj": 1433, + "13.v_proj": 1433, + "13.o_proj": 1433, + "13.gate_proj": 2089, + "13.up_proj": 2089, + "13.down_proj": 2089, + "14.q_proj": 1433, + "14.k_proj": 1433, + "14.v_proj": 1433, + "14.o_proj": 1433, + "14.gate_proj": 2089, + "14.up_proj": 2089, + "14.down_proj": 2089, + "15.q_proj": 1433, + "15.k_proj": 1433, + "15.v_proj": 1433, + "15.o_proj": 1433, + "15.gate_proj": 2089, + "15.up_proj": 2089, + "15.down_proj": 2089, + "16.q_proj": 1433, + "16.k_proj": 1433, + "16.v_proj": 1433, + "16.o_proj": 1433, + "16.gate_proj": 2089, + "16.up_proj": 2089, + "16.down_proj": 2089, + "17.q_proj": 1433, + "17.k_proj": 1433, + "17.v_proj": 1433, + "17.o_proj": 1433, + "17.gate_proj": 2089, + "17.up_proj": 2089, + "17.down_proj": 2089, + "18.q_proj": 1433, + "18.k_proj": 1433, + "18.v_proj": 1433, + "18.o_proj": 1433, + "18.gate_proj": 2089, + "18.up_proj": 2089, + "18.down_proj": 2089, + "19.q_proj": 1433, + "19.k_proj": 1433, + "19.v_proj": 1433, + "19.o_proj": 1433, + "19.gate_proj": 2089, + "19.up_proj": 2089, + "19.down_proj": 2089, + "20.q_proj": 1433, + "20.k_proj": 1433, + "20.v_proj": 1433, + "20.o_proj": 1433, + "20.gate_proj": 2089, + "20.up_proj": 2089, + "20.down_proj": 2089, + "21.q_proj": 1433, + "21.k_proj": 1433, + "21.v_proj": 1433, + "21.o_proj": 1433, + "21.gate_proj": 2089, + "21.up_proj": 2089, + "21.down_proj": 2089, + "22.q_proj": 1433, + "22.k_proj": 1433, + "22.v_proj": 1433, + "22.o_proj": 1433, + "22.gate_proj": 2089, + "22.up_proj": 2089, + "22.down_proj": 2089, + "23.q_proj": 1433, + "23.k_proj": 1433, + "23.v_proj": 1433, + "23.o_proj": 1433, + "23.gate_proj": 2089, + "23.up_proj": 2089, + "23.down_proj": 2089, + "24.q_proj": 1433, + "24.k_proj": 1433, + "24.v_proj": 1433, + "24.o_proj": 1433, + "24.gate_proj": 2089, + "24.up_proj": 2089, + "24.down_proj": 2089, + "25.q_proj": 1433, + "25.k_proj": 1433, + "25.v_proj": 1433, + "25.o_proj": 1433, + "25.gate_proj": 2089, + "25.up_proj": 2089, + "25.down_proj": 2089, + "26.q_proj": 1433, + "26.k_proj": 1433, + "26.v_proj": 1433, + "26.o_proj": 1433, + "26.gate_proj": 2089, + "26.up_proj": 2089, + "26.down_proj": 2089, + "27.q_proj": 1433, + "27.k_proj": 1433, + "27.v_proj": 1433, + "27.o_proj": 1433, + "27.gate_proj": 2089, + "27.up_proj": 2089, + "27.down_proj": 2089, + "28.q_proj": 1433, + "28.k_proj": 1433, + "28.v_proj": 1433, + "28.o_proj": 1433, + "28.gate_proj": 2089, + "28.up_proj": 2089, + "28.down_proj": 2089, + "29.q_proj": 1433, + "29.k_proj": 1433, + "29.v_proj": 1433, + "29.o_proj": 1433, + "29.gate_proj": 2089, + "29.up_proj": 2089, + "29.down_proj": 2089, + "30.q_proj": 1433, + "30.k_proj": 1433, + "30.v_proj": 1433, + "30.o_proj": 1433, + "30.gate_proj": 2089, + "30.up_proj": 2089, + "30.down_proj": 2089, + "31.q_proj": 1433, + "31.k_proj": 1433, + "31.v_proj": 1433, + "31.o_proj": 1433, + "31.gate_proj": 2089, + "31.up_proj": 2089, + "31.down_proj": 2089 + }, + "factored": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove30/truncated/factored.pt", + "folded": false, + "qkv_bias_restored": 0 +} \ No newline at end of file diff --git a/resmix/x_s_corpus.json b/resmix/x_s_corpus.json new file mode 100644 index 0000000..4215827 --- /dev/null +++ b/resmix/x_s_corpus.json @@ -0,0 +1,73 @@ +{ + "jbb_harm_full": { + "name": "jbb_harm_full", + "composition": "harm", + "span": "full", + "n_seqs": 11, + "seqlen": 2048, + "n_tokens": 22528, + "n_prompts_available": 100, + "n_special_per_seq": 15.545454978942871, + "source": "JailbreakBench harmful behaviours (Goal)", + "role": "harmful goal + dense model response", + "response_model": "meta-llama/Llama-2-7b-chat-hf", + "n_records": 96, + "n_kept": 96, + "min_response_chars": 20, + "dense_refusal_cue_fraction": 1.0, + "seed": 42, + "eval_overlap": { + "advbench": 11, + "strongreject": 3 + }, + "eval_overlap_subtracted": false, + "responses_from": "data/alpaca_cleaned_jbbsft_x10.json.refusals.json", + "n_units": 96, + "mean_tokens_per_unit": 264.9270833333333 + }, + "jbb_mix_full": { + "name": "jbb_mix_full", + "composition": "jbb harmful refusals + jbb benign answers", + "span": "full", + "n_seqs": 23, + "seqlen": 2048, + "n_tokens": 47104, + "n_special_per_seq": 15.0, + "harm": { + "source": "/home/jeesup0103/svd_safety/data/alpaca_cleaned_jbbsft_x10.json.refusals.json", + "n_units": 96, + "distinct_tokens": 25433, + "repeat": 1 + }, + "benign": { + "source": "/home/jeesup0103/svd_safety/data/safety_calib/jbb_benign_dense_responses.json", + "n_records": 100, + "n_refused_dropped": 54, + "n_units": 46, + "distinct_tokens": 12935, + "repeat": 2 + }, + "harm_token_share_before_packing": 0.4957, + "eval_overlap_note": "JBB harmful shares 11 AdvBench and 3 StrongREJECT goals (kept, as in the jbb arm)", + "seed": 42, + "n_units": 188, + "mean_tokens_per_unit": 272.88829787234044 + }, + "harm_full": { + "name": "harm_full", + "composition": "harm", + "span": "user turn + dense model response", + "n_seqs": 451, + "seqlen": 2048, + "n_tokens": 923648, + "n_prompts_available": 4018, + "n_special_per_seq": 16.81374740600586, + "n_units": 4018, + "mean_tokens_per_unit": 244.0955699352912, + "response_model": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", + "n_records": 4018, + "n_kept": 4018, + "min_response_chars": 20, + "dense_refusal_cue_fraction": 0.9624191139870583 + } +} \ No newline at end of file diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..451134b --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..6c00c74 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..6e95a65 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,44 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": true, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "", + "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\n' + system_message + '\\n<>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": false, + "model_max_length": 1000000000000000019884624838656, + "pad_token": null, + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +}