commit ee8466ad0f8181cca1f9c46fdc385e49248b029a Author: ModelHub XC Date: Thu Oct 22 13:59:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Jeesup/svd-safety-l2_base_k0_a1p0_free_remove10 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..51089e2 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,126 @@ +LLAMA 2 COMMUNITY LICENSE AGREEMENT +Llama 2 Version Release Date: July 18, 2023 + +"Agreement" means the terms and conditions for use, reproduction, distribution and +modification of the Llama Materials set forth herein. + +"Documentation" means the specifications, manuals and documentation +accompanying Llama 2 distributed by Meta at ai.meta.com/resources/models-and- +libraries/llama-downloads/. + +"Licensee" or "you" means you, or your employer or any other person or entity (if +you are entering into this Agreement on such person or entity's behalf), of the age +required under applicable laws, rules or regulations to provide legal consent and that +has legal authority to bind your employer or such other person or entity if you are +entering in this Agreement on their behalf. + +"Llama 2" means the foundational large language models and software and +algorithms, including machine-learning model code, trained model weights, +inference-enabling code, training-enabling code, fine-tuning enabling code and other +elements of the foregoing distributed by Meta at ai.meta.com/resources/models-and- +libraries/llama-downloads/. + +"Llama Materials" means, collectively, Meta's proprietary Llama 2 and +Documentation (and any portion thereof) made available under this Agreement. + +"Meta" or "we" means Meta Platforms Ireland Limited (if you are located in or, if you +are an entity, your principal place of business is in the EEA or Switzerland) and Meta +Platforms, Inc. (if you are located outside of the EEA or Switzerland). + +By clicking "I Accept" below or by using or distributing any portion or element of the +Llama Materials, you agree to be bound by this Agreement. + +1. License Rights and Redistribution. + + a. Grant of Rights. You are granted a non-exclusive, worldwide, non- +transferable and royalty-free limited license under Meta's intellectual property or +other rights owned by Meta embodied in the Llama Materials to use, reproduce, +distribute, copy, create derivative works of, and make modifications to the Llama +Materials. + + b. Redistribution and Use. + + i. If you distribute or make the Llama Materials, or any derivative works +thereof, available to a third party, you shall provide a copy of this Agreement to such +third party. + ii. If you receive Llama Materials, or any derivative works thereof, from +a Licensee as part of an integrated end user product, then Section 2 of this +Agreement will not apply to you. + + iii. You must retain in all copies of the Llama Materials that you +distribute the following attribution notice within a "Notice" text file distributed as a +part of such copies: "Llama 2 is licensed under the LLAMA 2 Community License, +Copyright (c) Meta Platforms, Inc. All Rights Reserved." + + iv. Your use of the Llama Materials must comply with applicable laws +and regulations (including trade compliance laws and regulations) and adhere to the +Acceptable Use Policy for the Llama Materials (available at +https://ai.meta.com/llama/use-policy), which is hereby incorporated by reference into +this Agreement. + + v. You will not use the Llama Materials or any output or results of the +Llama Materials to improve any other large language model (excluding Llama 2 or +derivative works thereof). + +2. Additional Commercial Terms. If, on the Llama 2 version release date, the +monthly active users of the products or services made available by or for Licensee, +or Licensee's affiliates, is greater than 700 million monthly active users in the +preceding calendar month, you must request a license from Meta, which Meta may +grant to you in its sole discretion, and you are not authorized to exercise any of the +rights under this Agreement unless or until Meta otherwise expressly grants you +such rights. + +3. Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE +LLAMA MATERIALS AND ANY OUTPUT AND RESULTS THEREFROM ARE +PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY +WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR +FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE +FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING +THE LLAMA MATERIALS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR +USE OF THE LLAMA MATERIALS AND ANY OUTPUT AND RESULTS. + +4. Limitation of Liability. IN NO EVENT WILL META OR ITS AFFILIATES BE +LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, +NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS +AGREEMENT, FOR ANY LOST PROFITS OR ANY INDIRECT, SPECIAL, +CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN +IF META OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF +ANY OF THE FOREGOING. + +5. Intellectual Property. + + a. No trademark licenses are granted under this Agreement, and in +connection with the Llama Materials, neither Meta nor Licensee may use any name +or mark owned by or associated with the other or any of its affiliates, except as +required for reasonable and customary use in describing and redistributing the +Llama Materials. + + b. Subject to Meta's ownership of Llama Materials and derivatives made by or +for Meta, with respect to any derivative works and modifications of the Llama +Materials that are made by you, as between you and Meta, you are and will be the +owner of such derivative works and modifications. + + c. If you institute litigation or other proceedings against Meta or any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the Llama +Materials or Llama 2 outputs or results, or any portion of any of the foregoing, +constitutes infringement of intellectual property or other rights owned or licensable +by you, then any licenses granted to you under this Agreement shall terminate as of +the date such litigation or claim is filed or instituted. You will indemnify and hold +harmless Meta from and against any claim by any third party arising out of or related +to your use or distribution of the Llama Materials. + +6. Term and Termination. The term of this Agreement will commence upon your +acceptance of this Agreement or access to the Llama Materials and will continue in +full force and effect until terminated in accordance with the terms and conditions +herein. Meta may terminate this Agreement if you are in breach of any term or +condition of this Agreement. Upon termination of this Agreement, you shall delete +and cease use of the Llama Materials. Sections 3, 4 and 7 shall survive the +termination of this Agreement. + +7. Governing Law and Jurisdiction. This Agreement will be governed and +construed under the laws of the State of California without regard to choice of law +principles, and the UN Convention on Contracts for the International Sale of Goods +does not apply to this Agreement. The courts of California shall have exclusive +jurisdiction of any dispute arising out of this Agreement. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..20ac0f3 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +license: llama2 +base_model: meta-llama/Llama-2-7b-chat-hf +library_name: transformers +pipeline_tag: text-generation +tags: +- llama2 +- svd +- compression +- safety +- interpretability +--- + +# svd-safety-l2_base_k0_a1p0_free_remove10 + +A Llama-2-7b-chat checkpoint compressed with SVD-LLM to **90.0% of dense +parameters**, then given a **0.0% parameter budget** of restored SVD +components selected by the **`unknown`** rule. + +This is a research artifact from a study of how SVD compression damages safety +behaviour and which component-selection rule best repairs it. It is one cell of a +grid over selection rules and budgets; it is **not** a general-purpose chat model. + +## Provenance + +| field | value | +|---|---| +| base (uncompressed) | `meta-llama/Llama-2-7b-chat-hf` | +| compression | SVD-LLM, 10.00% of parameters removed | +| selection rule | `unknown` | +| restore budget | 0.000% of dense parameters | +| components restored | 0 | +| components swapped out | 0 | +| resulting parameter fraction | 0.8998 | +| seed | 42 | + +## Measured + +| metric | value | +|---|---| +| AdvBench ASR (HarmBench judge) | 0.0154 | +| StrongREJECT ASR (HarmBench judge) | 0.0096 | +| Macro over-refusal (WildGuard) | 0.3860 | +| WikiText-2 perplexity | 8.0660 | + + +## Intended use and limitations + +This checkpoint exists to measure safety/utility trade-offs under compression. +Several arms in the grid are **deliberately safety-degraded** relative to +Llama-2-7b-chat: compression alone raises attack-success rate, and the point of +the study is to quantify that and test recovery. Treat any given cell as an +experimental subject, not as a deployable assistant, and evaluate it yourself +before drawing conclusions from it. + +## Licence + +Llama 2 Community License. `LICENSE.txt` and `USE_POLICY.md` are included in this repository, and use of this derivative is bound by them. Built with Llama 2. diff --git a/USE_POLICY.md b/USE_POLICY.md new file mode 100644 index 0000000..abbcc19 --- /dev/null +++ b/USE_POLICY.md @@ -0,0 +1,50 @@ +# Llama 2 Acceptable Use Policy + +Meta is committed to promoting safe and fair use of its tools and features, including Llama 2. If you access or use Llama 2, you agree to this Acceptable Use Policy (“Policy”). The most recent copy of this policy can be found at [ai.meta.com/llama/use-policy](http://ai.meta.com/llama/use-policy). + +## Prohibited Uses +We want everyone to use Llama 2 safely and responsibly. You agree you will not use, or allow others to use, Llama 2 to: + +1. Violate the law or others’ rights, including to: + 1. Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content, such as: + 1. Violence or terrorism + 2. Exploitation or harm to children, including the solicitation, creation, acquisition, or dissemination of child exploitative content or failure to report Child Sexual Abuse Material + 3. Human trafficking, exploitation, and sexual violence + 4. The illegal distribution of information or materials to minors, including obscene materials, or failure to employ legally required age-gating in connection with such information or materials. + 5. Sexual solicitation + 6. Any other criminal activity + 2. Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals + 3. Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services + 4. Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices + 5. Collect, process, disclose, generate, or infer health, demographic, or other sensitive personal or private information about individuals without rights and consents required by applicable laws + 6. Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any products or services using the Llama 2 Materials + 7. Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system + + + +2. Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including use of Llama 2 related to the following: + 1. Military, warfare, nuclear industries or applications, espionage, use for materials or activities that are subject to the International Traffic Arms Regulations (ITAR) maintained by the United States Department of State + 2. Guns and illegal weapons (including weapon development) + 3. Illegal drugs and regulated/controlled substances + 4. Operation of critical infrastructure, transportation technologies, or heavy machinery + 5. Self-harm or harm to others, including suicide, cutting, and eating disorders + 6. Any content intended to incite or promote violence, abuse, or any infliction of bodily harm to an individual + + + +3. Intentionally deceive or mislead others, including use of Llama 2 related to the following: + 1. Generating, promoting, or furthering fraud or the creation or promotion of disinformation + 2. Generating, promoting, or furthering defamatory content, including the creation of defamatory statements, images, or other content + 3. Generating, promoting, or further distributing spam + 4. Impersonating another individual without consent, authorization, or legal right + 5. Representing that the use of Llama 2 or outputs are human-generated + 6. Generating or facilitating false online engagement, including fake reviews and other means of fake online engagement +4. Fail to appropriately disclose to end users any known dangers of your AI system + +Please report any violation of this Policy, software “bug,” or other problems that could lead to a violation of this Policy through one of the following means: + +* Reporting issues with the model: [github.com/facebookresearch/llama](http://github.com/facebookresearch/llama) +* Reporting risky content generated by the model: [developers.facebook.com/llama_output_feedback](http://developers.facebook.com/llama_output_feedback) +* Reporting bugs and security concerns: [facebook.com/whitehat/info](http://facebook.com/whitehat/info) +* Reporting violations of the Acceptable Use Policy or unlicensed uses of Llama: [LlamaUseReport@meta.com](mailto:LlamaUseReport@meta.com) + diff --git a/compression.json b/compression.json new file mode 100644 index 0000000..3f64048 --- /dev/null +++ b/compression.json @@ -0,0 +1,2060 @@ +{ + "source": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", + "upstream": "AIoT-MLSys-Lab/SVD-LLM", + "stage": "finetuned", + "ratio_removed": 0.1, + "internal_ratio_kept": 0.9, + "from_pt": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove10/finetune_v/merge.pt", + "biases_verified": 0, + "mean_rel_err": 0.36113157379440963, + "max_rel_err": 1.207053780555725, + "param_fraction": 0.8998112653821243, + "ranks": { + "0.q_proj": 1843, + "0.k_proj": 1843, + "0.v_proj": 1843, + "0.o_proj": 1843, + "0.gate_proj": 2686, + "0.up_proj": 2686, + "0.down_proj": 2686, + "1.q_proj": 1843, + "1.k_proj": 1843, + "1.v_proj": 1843, + "1.o_proj": 1843, + "1.gate_proj": 2686, + "1.up_proj": 2686, + "1.down_proj": 2686, + "2.q_proj": 1843, + "2.k_proj": 1843, + "2.v_proj": 1843, + "2.o_proj": 1843, + "2.gate_proj": 2686, + "2.up_proj": 2686, + "2.down_proj": 2686, + "3.q_proj": 1843, + "3.k_proj": 1843, + "3.v_proj": 1843, + "3.o_proj": 1843, + "3.gate_proj": 2686, + "3.up_proj": 2686, + "3.down_proj": 2686, + "4.q_proj": 1843, + "4.k_proj": 1843, + "4.v_proj": 1843, + "4.o_proj": 1843, + "4.gate_proj": 2686, + "4.up_proj": 2686, + "4.down_proj": 2686, + "5.q_proj": 1843, + "5.k_proj": 1843, + "5.v_proj": 1843, + "5.o_proj": 1843, + "5.gate_proj": 2686, + "5.up_proj": 2686, + "5.down_proj": 2686, + "6.q_proj": 1843, + "6.k_proj": 1843, + "6.v_proj": 1843, + "6.o_proj": 1843, + "6.gate_proj": 2686, + "6.up_proj": 2686, + "6.down_proj": 2686, + "7.q_proj": 1843, + "7.k_proj": 1843, + "7.v_proj": 1843, + "7.o_proj": 1843, + "7.gate_proj": 2686, + "7.up_proj": 2686, + "7.down_proj": 2686, + "8.q_proj": 1843, + "8.k_proj": 1843, + "8.v_proj": 1843, + "8.o_proj": 1843, + "8.gate_proj": 2686, + "8.up_proj": 2686, + "8.down_proj": 2686, + "9.q_proj": 1843, + "9.k_proj": 1843, + "9.v_proj": 1843, + "9.o_proj": 1843, + "9.gate_proj": 2686, + "9.up_proj": 2686, + "9.down_proj": 2686, + "10.q_proj": 1843, + "10.k_proj": 1843, + "10.v_proj": 1843, + "10.o_proj": 1843, + "10.gate_proj": 2686, + "10.up_proj": 2686, + "10.down_proj": 2686, + "11.q_proj": 1843, + "11.k_proj": 1843, + "11.v_proj": 1843, + "11.o_proj": 1843, + "11.gate_proj": 2686, + "11.up_proj": 2686, + "11.down_proj": 2686, + "12.q_proj": 1843, + "12.k_proj": 1843, + "12.v_proj": 1843, + "12.o_proj": 1843, + "12.gate_proj": 2686, + "12.up_proj": 2686, + "12.down_proj": 2686, + "13.q_proj": 1843, + "13.k_proj": 1843, + "13.v_proj": 1843, + "13.o_proj": 1843, + "13.gate_proj": 2686, + "13.up_proj": 2686, + "13.down_proj": 2686, + "14.q_proj": 1843, + "14.k_proj": 1843, + "14.v_proj": 1843, + "14.o_proj": 1843, + "14.gate_proj": 2686, + "14.up_proj": 2686, + "14.down_proj": 2686, + "15.q_proj": 1843, + "15.k_proj": 1843, + "15.v_proj": 1843, + "15.o_proj": 1843, + "15.gate_proj": 2686, + "15.up_proj": 2686, + "15.down_proj": 2686, + "16.q_proj": 1843, + "16.k_proj": 1843, + "16.v_proj": 1843, + "16.o_proj": 1843, + "16.gate_proj": 2686, + "16.up_proj": 2686, + "16.down_proj": 2686, + "17.q_proj": 1843, + "17.k_proj": 1843, + "17.v_proj": 1843, + "17.o_proj": 1843, + "17.gate_proj": 2686, + "17.up_proj": 2686, + "17.down_proj": 2686, + "18.q_proj": 1843, + "18.k_proj": 1843, + "18.v_proj": 1843, + "18.o_proj": 1843, + "18.gate_proj": 2686, + "18.up_proj": 2686, + "18.down_proj": 2686, + "19.q_proj": 1843, + "19.k_proj": 1843, + "19.v_proj": 1843, + "19.o_proj": 1843, + "19.gate_proj": 2686, + "19.up_proj": 2686, + "19.down_proj": 2686, + "20.q_proj": 1843, + "20.k_proj": 1843, + "20.v_proj": 1843, + "20.o_proj": 1843, + "20.gate_proj": 2686, + "20.up_proj": 2686, + "20.down_proj": 2686, + "21.q_proj": 1843, + "21.k_proj": 1843, + "21.v_proj": 1843, + "21.o_proj": 1843, + "21.gate_proj": 2686, + "21.up_proj": 2686, + "21.down_proj": 2686, + "22.q_proj": 1843, + "22.k_proj": 1843, + "22.v_proj": 1843, + "22.o_proj": 1843, + "22.gate_proj": 2686, + "22.up_proj": 2686, + "22.down_proj": 2686, + "23.q_proj": 1843, + "23.k_proj": 1843, + "23.v_proj": 1843, + "23.o_proj": 1843, + "23.gate_proj": 2686, + "23.up_proj": 2686, + "23.down_proj": 2686, + "24.q_proj": 1843, + "24.k_proj": 1843, + "24.v_proj": 1843, + "24.o_proj": 1843, + "24.gate_proj": 2686, + "24.up_proj": 2686, + "24.down_proj": 2686, + "25.q_proj": 1843, + "25.k_proj": 1843, + "25.v_proj": 1843, + "25.o_proj": 1843, + "25.gate_proj": 2686, + "25.up_proj": 2686, + "25.down_proj": 2686, + "26.q_proj": 1843, + "26.k_proj": 1843, + "26.v_proj": 1843, + "26.o_proj": 1843, + "26.gate_proj": 2686, + "26.up_proj": 2686, + "26.down_proj": 2686, + "27.q_proj": 1843, + "27.k_proj": 1843, + "27.v_proj": 1843, + "27.o_proj": 1843, + "27.gate_proj": 2686, + "27.up_proj": 2686, + "27.down_proj": 2686, + "28.q_proj": 1843, + "28.k_proj": 1843, + "28.v_proj": 1843, + "28.o_proj": 1843, + "28.gate_proj": 2686, + "28.up_proj": 2686, + "28.down_proj": 2686, + "29.q_proj": 1843, + "29.k_proj": 1843, + "29.v_proj": 1843, + "29.o_proj": 1843, + "29.gate_proj": 2686, + "29.up_proj": 2686, + "29.down_proj": 2686, + "30.q_proj": 1843, + "30.k_proj": 1843, + "30.v_proj": 1843, + "30.o_proj": 1843, + "30.gate_proj": 2686, + "30.up_proj": 2686, + "30.down_proj": 2686, + "31.q_proj": 1843, + "31.k_proj": 1843, + "31.v_proj": 1843, + "31.o_proj": 1843, + "31.gate_proj": 2686, + "31.up_proj": 2686, + "31.down_proj": 2686 + }, + "rel_err": { + "0.q_proj": 0.19462965428829193, + "0.k_proj": 0.24923677742481232, + "0.v_proj": 0.24044226109981537, + "0.o_proj": 0.24431376159191132, + "0.gate_proj": 0.28850123286247253, + "0.up_proj": 0.29543015360832214, + "0.down_proj": 0.3452581465244293, + "1.q_proj": 0.0892096757888794, + "1.k_proj": 0.07323329895734787, + "1.v_proj": 0.3366854190826416, + "1.o_proj": 0.26621776819229126, + "1.gate_proj": 0.26865020394325256, + "1.up_proj": 0.30451568961143494, + "1.down_proj": 0.3906521797180176, + "2.q_proj": 0.5516203045845032, + "2.k_proj": 1.207053780555725, + "2.v_proj": 0.4859524071216583, + "2.o_proj": 0.4095371663570404, + "2.gate_proj": 0.32742226123809814, + "2.up_proj": 0.3512292504310608, + "2.down_proj": 0.38645046949386597, + "3.q_proj": 0.25567734241485596, + "3.k_proj": 0.2517416179180145, + "3.v_proj": 0.3778376281261444, + "3.o_proj": 0.40221676230430603, + "3.gate_proj": 0.3702404797077179, + "3.up_proj": 0.3856930434703827, + "3.down_proj": 0.3590887486934662, + "4.q_proj": 0.24836672842502594, + "4.k_proj": 0.24987998604774475, + "4.v_proj": 0.3751924932003021, + "4.o_proj": 0.4040300250053406, + "4.gate_proj": 0.37244343757629395, + "4.up_proj": 0.3922712802886963, + "4.down_proj": 0.36263343691825867, + "5.q_proj": 0.29160168766975403, + "5.k_proj": 0.26649215817451477, + "5.v_proj": 0.3769007921218872, + "5.o_proj": 0.41241952776908875, + "5.gate_proj": 0.3697393238544464, + "5.up_proj": 0.39346516132354736, + "5.down_proj": 0.3549274802207947, + "6.q_proj": 0.2544267773628235, + "6.k_proj": 0.2589779198169708, + "6.v_proj": 0.36512354016304016, + "6.o_proj": 0.3814370930194855, + "6.gate_proj": 0.3652000427246094, + "6.up_proj": 0.3938547372817993, + "6.down_proj": 0.35428211092948914, + "7.q_proj": 0.25830814242362976, + "7.k_proj": 0.26263171434402466, + "7.v_proj": 0.35945364832878113, + "7.o_proj": 0.39563044905662537, + "7.gate_proj": 0.3632635474205017, + "7.up_proj": 0.3941986560821533, + "7.down_proj": 0.3489327132701874, + "8.q_proj": 0.2653649151325226, + "8.k_proj": 0.27433836460113525, + "8.v_proj": 0.3488828241825104, + "8.o_proj": 0.3927844762802124, + "8.gate_proj": 0.36154937744140625, + "8.up_proj": 0.3910050094127655, + "8.down_proj": 0.3464740216732025, + "9.q_proj": 0.2827792763710022, + "9.k_proj": 0.2866414189338684, + "9.v_proj": 0.34705373644828796, + "9.o_proj": 0.39915502071380615, + "9.gate_proj": 0.3595671057701111, + "9.up_proj": 0.3867504894733429, + "9.down_proj": 0.3463095724582672, + "10.q_proj": 0.2654861807823181, + "10.k_proj": 0.2705942690372467, + "10.v_proj": 0.344517320394516, + "10.o_proj": 0.3558924198150635, + "10.gate_proj": 0.35533344745635986, + "10.up_proj": 0.383311927318573, + "10.down_proj": 0.3432478606700897, + "11.q_proj": 0.4137333035469055, + "11.k_proj": 0.5051218271255493, + "11.v_proj": 0.3657678961753845, + "11.o_proj": 0.34954679012298584, + "11.gate_proj": 0.35416820645332336, + "11.up_proj": 0.3829926550388336, + "11.down_proj": 0.34822148084640503, + "12.q_proj": 0.2762104868888855, + "12.k_proj": 0.2754881680011749, + "12.v_proj": 0.34544119238853455, + "12.o_proj": 0.3824060559272766, + "12.gate_proj": 0.3575476408004761, + "12.up_proj": 0.38287779688835144, + "12.down_proj": 0.3477227985858917, + "13.q_proj": 0.2813850939273834, + "13.k_proj": 0.2932816445827484, + "13.v_proj": 0.36768674850463867, + "13.o_proj": 0.3750237226486206, + "13.gate_proj": 0.35995373129844666, + "13.up_proj": 0.3854982554912567, + "13.down_proj": 0.35798031091690063, + "14.q_proj": 0.2710683345794678, + "14.k_proj": 0.2745169997215271, + "14.v_proj": 0.3610003888607025, + "14.o_proj": 0.37273406982421875, + "14.gate_proj": 0.35924577713012695, + "14.up_proj": 0.38422349095344543, + "14.down_proj": 0.3590209186077118, + "15.q_proj": 0.2833070755004883, + "15.k_proj": 0.28545209765434265, + "15.v_proj": 0.3664618134498596, + "15.o_proj": 0.37796980142593384, + "15.gate_proj": 0.3634200692176819, + "15.up_proj": 0.3882993161678314, + "15.down_proj": 0.37039846181869507, + "16.q_proj": 0.2815943956375122, + "16.k_proj": 0.28404316306114197, + "16.v_proj": 0.3875197470188141, + "16.o_proj": 0.4045898914337158, + "16.gate_proj": 0.36150580644607544, + "16.up_proj": 0.3858390152454376, + "16.down_proj": 0.37960028648376465, + "17.q_proj": 0.3100113868713379, + "17.k_proj": 0.30941879749298096, + "17.v_proj": 0.40104714035987854, + "17.o_proj": 0.41341009736061096, + "17.gate_proj": 0.3639795780181885, + "17.up_proj": 0.3859553635120392, + "17.down_proj": 0.37684857845306396, + "18.q_proj": 0.32933807373046875, + "18.k_proj": 0.3337570130825043, + "18.v_proj": 0.4073031544685364, + "18.o_proj": 0.44279807806015015, + "18.gate_proj": 0.37023791670799255, + "18.up_proj": 0.391116738319397, + "18.down_proj": 0.3781541883945465, + "19.q_proj": 0.31660863757133484, + "19.k_proj": 0.3154717683792114, + "19.v_proj": 0.3943529725074768, + "19.o_proj": 0.4314393699169159, + "19.gate_proj": 0.3739177882671356, + "19.up_proj": 0.392498254776001, + "19.down_proj": 0.37863025069236755, + "20.q_proj": 0.30987852811813354, + "20.k_proj": 0.31104883551597595, + "20.v_proj": 0.4043391942977905, + "20.o_proj": 0.42956486344337463, + "20.gate_proj": 0.37242448329925537, + "20.up_proj": 0.3892928957939148, + "20.down_proj": 0.3864456117153168, + "21.q_proj": 0.32015499472618103, + "21.k_proj": 0.3221917748451233, + "21.v_proj": 0.4016200602054596, + "21.o_proj": 0.4437079429626465, + "21.gate_proj": 0.373360276222229, + "21.up_proj": 0.38891369104385376, + "21.down_proj": 0.38294145464897156, + "22.q_proj": 0.32665300369262695, + "22.k_proj": 0.33194848895072937, + "22.v_proj": 0.41098883748054504, + "22.o_proj": 0.44017305970191956, + "22.gate_proj": 0.37318068742752075, + "22.up_proj": 0.38812994956970215, + "22.down_proj": 0.38084593415260315, + "23.q_proj": 0.3585113286972046, + "23.k_proj": 0.35559317469596863, + "23.v_proj": 0.4094955623149872, + "23.o_proj": 0.4310721158981323, + "23.gate_proj": 0.37551355361938477, + "23.up_proj": 0.3899531960487366, + "23.down_proj": 0.37517276406288147, + "24.q_proj": 0.3101503252983093, + "24.k_proj": 0.3136228322982788, + "24.v_proj": 0.40340301394462585, + "24.o_proj": 0.43703603744506836, + "24.gate_proj": 0.3759905695915222, + "24.up_proj": 0.3900173306465149, + "24.down_proj": 0.37048637866973877, + "25.q_proj": 0.35527220368385315, + "25.k_proj": 0.35458481311798096, + "25.v_proj": 0.4108697474002838, + "25.o_proj": 0.43194887042045593, + "25.gate_proj": 0.3782326281070709, + "25.up_proj": 0.3914431929588318, + "25.down_proj": 0.36568114161491394, + "26.q_proj": 0.32080888748168945, + "26.k_proj": 0.3287152647972107, + "26.v_proj": 0.4006011188030243, + "26.o_proj": 0.4577499330043793, + "26.gate_proj": 0.37910711765289307, + "26.up_proj": 0.39304906129837036, + "26.down_proj": 0.3653787076473236, + "27.q_proj": 0.36237528920173645, + "27.k_proj": 0.3694652020931244, + "27.v_proj": 0.41660192608833313, + "27.o_proj": 0.43260541558265686, + "27.gate_proj": 0.38131773471832275, + "27.up_proj": 0.3951241672039032, + "27.down_proj": 0.3652697503566742, + "28.q_proj": 0.3470246493816376, + "28.k_proj": 0.35592564940452576, + "28.v_proj": 0.41344523429870605, + "28.o_proj": 0.43140166997909546, + "28.gate_proj": 0.3822683095932007, + "28.up_proj": 0.394208163022995, + "28.down_proj": 0.3720575273036957, + "29.q_proj": 0.32196155190467834, + "29.k_proj": 0.3285108506679535, + "29.v_proj": 0.40625032782554626, + "29.o_proj": 0.4305853247642517, + "29.gate_proj": 0.3812209963798523, + "29.up_proj": 0.3908163011074066, + "29.down_proj": 0.37770071625709534, + "30.q_proj": 0.3496403992176056, + "30.k_proj": 0.3554087281227112, + "30.v_proj": 0.4186227023601532, + "30.o_proj": 0.4390658438205719, + "30.gate_proj": 0.37650272250175476, + "30.up_proj": 0.38091957569122314, + "30.down_proj": 0.38477763533592224, + "31.q_proj": 0.3052699565887451, + "31.k_proj": 0.311748743057251, + "31.v_proj": 0.4005729556083679, + "31.o_proj": 0.4107411801815033, + "31.gate_proj": 0.36805716156959534, + "31.up_proj": 0.3683338165283203, + "31.down_proj": 0.38238590955734253 + }, + "whitening": { + "lam": 0.0, + "norm": "token", + "ridge": 0.0, + "token_mask": "nospecial", + "utility_cache": "/lustre/jeesup0103/svd-safety/work/whiten_cache/utility_wikitext256", + "utility_calib": "wikitext2:256", + "utility_tokens": 522240 + }, + "rank_by": "resmix", + "resmix_k": 0.0, + "resmix_k_abs": null, + "ranking": { + "n_matrices": 224, + "alpha": 1.0, + "k_s_total": 0, + "k_s_frac_of_rank": 0.0, + "utility_error_ratio": 1.0, + "resmix_k": 0.0, + "resmix_k_abs": null, + "resmix_alpha": 1.0 + }, + "ranking_per_matrix": { + "0.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 10.785678176430773, + "T_cal": 10.785678176430773 + }, + "0.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 9.576390170908583, + "T_cal": 9.576390170908583 + }, + "0.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 851.6657157551547, + "T_cal": 851.6657157551547 + }, + "0.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 0.9780601726363409, + "T_cal": 0.9780601726363409 + }, + "0.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 89500.30917201788, + "T_cal": 89500.30917201788 + }, + "0.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 92107.91324240528, + "T_cal": 92107.91324240528 + }, + "0.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 2099.17623664463, + "T_cal": 2099.17623664463 + }, + "1.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 8719.575457896477, + "T_cal": 8719.575457896477 + }, + "1.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 6348.398214962525, + "T_cal": 6348.398214962525 + }, + "1.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 22890.31522699863, + "T_cal": 22890.31522699863 + }, + "1.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 98.1451973636449, + "T_cal": 98.1451973636449 + }, + "1.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 329123.88260184403, + "T_cal": 329123.88260184403 + }, + "1.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 376866.56578857085, + "T_cal": 376866.56578857085 + }, + "1.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 28736.74082056575, + "T_cal": 28736.74082056575 + }, + "2.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 643893.6614395604, + "T_cal": 643893.6614395604 + }, + "2.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 626112.885156469, + "T_cal": 626112.885156469 + }, + "2.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 876510.7086711043, + "T_cal": 876510.7086711043 + }, + "2.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 10899.537467294535, + "T_cal": 10899.537467294535 + }, + "2.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 1693703.440605626, + "T_cal": 1693703.440605626 + }, + "2.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 1696804.2039791269, + "T_cal": 1696804.2039791269 + }, + "2.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 85977.69509733199, + "T_cal": 85977.69509733199 + }, + "3.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 5783209.987075051, + "T_cal": 5783209.987075051 + }, + "3.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 6152009.641441202, + "T_cal": 6152009.641441202 + }, + "3.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 3739286.972493657, + "T_cal": 3739286.972493657 + }, + "3.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 19633.092639652994, + "T_cal": 19633.092639652994 + }, + "3.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 5126539.8926883275, + "T_cal": 5126539.8926883275 + }, + "3.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 4762765.7752103135, + "T_cal": 4762765.7752103135 + }, + "3.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 151902.5633119275, + "T_cal": 151902.5633119275 + }, + "4.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 5857197.810485085, + "T_cal": 5857197.810485085 + }, + "4.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 6178465.372496818, + "T_cal": 6178465.372496818 + }, + "4.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 3967419.9790056134, + "T_cal": 3967419.9790056134 + }, + "4.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 44046.95099997806, + "T_cal": 44046.95099997806 + }, + "4.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 7840698.699492722, + "T_cal": 7840698.699492722 + }, + "4.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 7040923.029561698, + "T_cal": 7040923.029561698 + }, + "4.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 280243.8009020678, + "T_cal": 280243.8009020678 + }, + "5.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 7545364.521031534, + "T_cal": 7545364.521031534 + }, + "5.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 8063765.271419729, + "T_cal": 8063765.271419729 + }, + "5.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 4943144.505550002, + "T_cal": 4943144.505550002 + }, + "5.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 58175.060551189556, + "T_cal": 58175.060551189556 + }, + "5.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 9873067.979936454, + "T_cal": 9873067.979936454 + }, + "5.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 8941202.13321339, + "T_cal": 8941202.13321339 + }, + "5.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 424218.746589352, + "T_cal": 424218.746589352 + }, + "6.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 10533861.771791458, + "T_cal": 10533861.771791458 + }, + "6.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 11390276.337218832, + "T_cal": 11390276.337218832 + }, + "6.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 6543838.750572076, + "T_cal": 6543838.750572076 + }, + "6.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 103800.18289654578, + "T_cal": 103800.18289654578 + }, + "6.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 12771164.726543933, + "T_cal": 12771164.726543933 + }, + "6.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 11397835.174568733, + "T_cal": 11397835.174568733 + }, + "6.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 604886.0140598821, + "T_cal": 604886.0140598821 + }, + "7.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 11844207.879055038, + "T_cal": 11844207.879055038 + }, + "7.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 12400266.770684075, + "T_cal": 12400266.770684075 + }, + "7.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 7136304.962956544, + "T_cal": 7136304.962956544 + }, + "7.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 148138.74464088996, + "T_cal": 148138.74464088996 + }, + "7.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 15039607.582688756, + "T_cal": 15039607.582688756 + }, + "7.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 13674887.732154544, + "T_cal": 13674887.732154544 + }, + "7.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 803697.9954367423, + "T_cal": 803697.9954367423 + }, + "8.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 12709352.285616899, + "T_cal": 12709352.285616899 + }, + "8.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 13722898.40869642, + "T_cal": 13722898.40869642 + }, + "8.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 7054155.237657632, + "T_cal": 7054155.237657632 + }, + "8.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 220494.06387088224, + "T_cal": 220494.06387088224 + }, + "8.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 15542306.874569666, + "T_cal": 15542306.874569666 + }, + "8.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 14794875.010815449, + "T_cal": 14794875.010815449 + }, + "8.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 982559.5498754395, + "T_cal": 982559.5498754395 + }, + "9.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 15184121.982329842, + "T_cal": 15184121.982329842 + }, + "9.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 16383592.258965734, + "T_cal": 16383592.258965734 + }, + "9.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 7869652.540471686, + "T_cal": 7869652.540471686 + }, + "9.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 326461.51250768616, + "T_cal": 326461.51250768616 + }, + "9.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 16271731.347973311, + "T_cal": 16271731.347973311 + }, + "9.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 15953925.992089037, + "T_cal": 15953925.992089037 + }, + "9.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 1187740.602015712, + "T_cal": 1187740.602015712 + }, + "10.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 13435789.945286755, + "T_cal": 13435789.945286755 + }, + "10.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 15122004.14128048, + "T_cal": 15122004.14128048 + }, + "10.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 7834700.253071317, + "T_cal": 7834700.253071317 + }, + "10.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 411647.8105691812, + "T_cal": 411647.8105691812 + }, + "10.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 16575456.584854057, + "T_cal": 16575456.584854057 + }, + "10.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 16757814.787526622, + "T_cal": 16757814.787526622 + }, + "10.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 1316334.097178603, + "T_cal": 1316334.097178603 + }, + "11.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 13930622.718331898, + "T_cal": 13930622.718331898 + }, + "11.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 15007569.816916827, + "T_cal": 15007569.816916827 + }, + "11.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 9711772.716010273, + "T_cal": 9711772.716010273 + }, + "11.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 391418.17282386805, + "T_cal": 391418.17282386805 + }, + "11.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 18402714.369628422, + "T_cal": 18402714.369628422 + }, + "11.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 19088554.366343856, + "T_cal": 19088554.366343856 + }, + "11.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 1495485.0911699792, + "T_cal": 1495485.0911699792 + }, + "12.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 18748398.866943624, + "T_cal": 18748398.866943624 + }, + "12.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 19914010.67244915, + "T_cal": 19914010.67244915 + }, + "12.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 10801728.444651768, + "T_cal": 10801728.444651768 + }, + "12.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 531758.5048293495, + "T_cal": 531758.5048293495 + }, + "12.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 19764191.62510787, + "T_cal": 19764191.62510787 + }, + "12.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 20794292.418899104, + "T_cal": 20794292.418899104 + }, + "12.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 1736375.6087208553, + "T_cal": 1736375.6087208553 + }, + "13.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 20292510.921908595, + "T_cal": 20292510.921908595 + }, + "13.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 22995179.546872757, + "T_cal": 22995179.546872757 + }, + "13.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 14077361.372142034, + "T_cal": 14077361.372142034 + }, + "13.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 540804.128938798, + "T_cal": 540804.128938798 + }, + "13.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 20812667.53851728, + "T_cal": 20812667.53851728 + }, + "13.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 22549834.16205047, + "T_cal": 22549834.16205047 + }, + "13.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 2137573.7105607325, + "T_cal": 2137573.7105607325 + }, + "14.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 18825714.543996416, + "T_cal": 18825714.543996416 + }, + "14.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 20136168.07042644, + "T_cal": 20136168.07042644 + }, + "14.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 13359328.927838562, + "T_cal": 13359328.927838562 + }, + "14.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 689326.1516168148, + "T_cal": 689326.1516168148 + }, + "14.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 22786515.866438046, + "T_cal": 22786515.866438046 + }, + "14.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 24728059.81873543, + "T_cal": 24728059.81873543 + }, + "14.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 2517011.435474057, + "T_cal": 2517011.435474057 + }, + "15.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 19993806.91956532, + "T_cal": 19993806.91956532 + }, + "15.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 21747526.803754296, + "T_cal": 21747526.803754296 + }, + "15.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 14665061.06597065, + "T_cal": 14665061.06597065 + }, + "15.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 817638.8098200579, + "T_cal": 817638.8098200579 + }, + "15.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 24911255.812186185, + "T_cal": 24911255.812186185 + }, + "15.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 27084609.74382753, + "T_cal": 27084609.74382753 + }, + "15.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 3193436.979680022, + "T_cal": 3193436.979680022 + }, + "16.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 20407912.52514683, + "T_cal": 20407912.52514683 + }, + "16.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 21974787.172197178, + "T_cal": 21974787.172197178 + }, + "16.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 18757995.0596334, + "T_cal": 18757995.0596334 + }, + "16.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1071887.20644118, + "T_cal": 1071887.20644118 + }, + "16.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 26989833.561512306, + "T_cal": 26989833.561512306 + }, + "16.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 28729017.121802475, + "T_cal": 28729017.121802475 + }, + "16.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 4155374.8409035476, + "T_cal": 4155374.8409035476 + }, + "17.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 25441887.973660365, + "T_cal": 25441887.973660365 + }, + "17.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 26466781.651458476, + "T_cal": 26466781.651458476 + }, + "17.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 20486151.87593612, + "T_cal": 20486151.87593612 + }, + "17.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 922322.9825638515, + "T_cal": 922322.9825638515 + }, + "17.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 32072077.254918516, + "T_cal": 32072077.254918516 + }, + "17.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 32978860.75940479, + "T_cal": 32978860.75940479 + }, + "17.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 4637992.671098169, + "T_cal": 4637992.671098169 + }, + "18.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 31773729.391317233, + "T_cal": 31773729.391317233 + }, + "18.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 34017678.001551084, + "T_cal": 34017678.001551084 + }, + "18.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 26689941.41991458, + "T_cal": 26689941.41991458 + }, + "18.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 938748.9824173935, + "T_cal": 938748.9824173935 + }, + "18.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 37749984.50054178, + "T_cal": 37749984.50054178 + }, + "18.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 37563549.93561437, + "T_cal": 37563549.93561437 + }, + "18.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 5552828.787711998, + "T_cal": 5552828.787711998 + }, + "19.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 28647353.80244772, + "T_cal": 28647353.80244772 + }, + "19.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 29289822.33732438, + "T_cal": 29289822.33732438 + }, + "19.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 25667160.26837124, + "T_cal": 25667160.26837124 + }, + "19.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1032724.8317759677, + "T_cal": 1032724.8317759677 + }, + "19.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 42595598.11038089, + "T_cal": 42595598.11038089 + }, + "19.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 41513706.856649965, + "T_cal": 41513706.856649965 + }, + "19.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 6309925.385078111, + "T_cal": 6309925.385078111 + }, + "20.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 28436851.374570876, + "T_cal": 28436851.374570876 + }, + "20.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 29759141.79584817, + "T_cal": 29759141.79584817 + }, + "20.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 28791771.883710474, + "T_cal": 28791771.883710474 + }, + "20.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1204475.571156554, + "T_cal": 1204475.571156554 + }, + "20.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 45059279.89659271, + "T_cal": 45059279.89659271 + }, + "20.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 43208565.658598244, + "T_cal": 43208565.658598244 + }, + "20.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 7628871.560222485, + "T_cal": 7628871.560222485 + }, + "21.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 32768398.130046345, + "T_cal": 32768398.130046345 + }, + "21.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 33770010.93750383, + "T_cal": 33770010.93750383 + }, + "21.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 33815038.04286927, + "T_cal": 33815038.04286927 + }, + "21.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1186598.6591508216, + "T_cal": 1186598.6591508216 + }, + "21.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 50949707.43289541, + "T_cal": 50949707.43289541 + }, + "21.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 47731695.09471324, + "T_cal": 47731695.09471324 + }, + "21.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 8489349.957435494, + "T_cal": 8489349.957435494 + }, + "22.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 38265573.64003351, + "T_cal": 38265573.64003351 + }, + "22.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 40043333.69933437, + "T_cal": 40043333.69933437 + }, + "22.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 37850521.35180683, + "T_cal": 37850521.35180683 + }, + "22.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1687446.2246034364, + "T_cal": 1687446.2246034364 + }, + "22.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 56684983.32320996, + "T_cal": 56684983.32320996 + }, + "22.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 52258393.50257513, + "T_cal": 52258393.50257513 + }, + "22.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 9890840.02519774, + "T_cal": 9890840.02519774 + }, + "23.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 51741264.84595457, + "T_cal": 51741264.84595457 + }, + "23.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 52444286.27460586, + "T_cal": 52444286.27460586 + }, + "23.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 48520675.52845783, + "T_cal": 48520675.52845783 + }, + "23.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1967040.5017786198, + "T_cal": 1967040.5017786198 + }, + "23.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 64495712.7115826, + "T_cal": 64495712.7115826 + }, + "23.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 59670810.45598352, + "T_cal": 59670810.45598352 + }, + "23.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 10872405.012488443, + "T_cal": 10872405.012488443 + }, + "24.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 37388957.17629345, + "T_cal": 37388957.17629345 + }, + "24.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 38682147.232396245, + "T_cal": 38682147.232396245 + }, + "24.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 46526989.69213562, + "T_cal": 46526989.69213562 + }, + "24.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 2045753.1178002125, + "T_cal": 2045753.1178002125 + }, + "24.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 71713721.48563766, + "T_cal": 71713721.48563766 + }, + "24.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 66417296.29878324, + "T_cal": 66417296.29878324 + }, + "24.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 11858175.648777504, + "T_cal": 11858175.648777504 + }, + "25.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 58150761.39573605, + "T_cal": 58150761.39573605 + }, + "25.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 60666231.365755424, + "T_cal": 60666231.365755424 + }, + "25.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 63400534.17654945, + "T_cal": 63400534.17654945 + }, + "25.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1833248.6132108124, + "T_cal": 1833248.6132108124 + }, + "25.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 79249577.4974488, + "T_cal": 79249577.4974488 + }, + "25.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 73455470.56279409, + "T_cal": 73455470.56279409 + }, + "25.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 12594951.21631376, + "T_cal": 12594951.21631376 + }, + "26.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 46642155.81444648, + "T_cal": 46642155.81444648 + }, + "26.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 49238810.41841708, + "T_cal": 49238810.41841708 + }, + "26.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 60145957.693433836, + "T_cal": 60145957.693433836 + }, + "26.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 2269621.8077429, + "T_cal": 2269621.8077429 + }, + "26.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 87286412.45394044, + "T_cal": 87286412.45394044 + }, + "26.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 81332614.31588593, + "T_cal": 81332614.31588593 + }, + "26.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 14127550.918734217, + "T_cal": 14127550.918734217 + }, + "27.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 69267576.00702935, + "T_cal": 69267576.00702935 + }, + "27.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 73067124.26488246, + "T_cal": 73067124.26488246 + }, + "27.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 69976263.48066255, + "T_cal": 69976263.48066255 + }, + "27.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 2894573.086826196, + "T_cal": 2894573.086826196 + }, + "27.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 96620978.22339875, + "T_cal": 96620978.22339875 + }, + "27.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 90349637.19218573, + "T_cal": 90349637.19218573 + }, + "27.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 16197426.509603195, + "T_cal": 16197426.509603195 + }, + "28.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 62137063.336638995, + "T_cal": 62137063.336638995 + }, + "28.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 66669531.8016634, + "T_cal": 66669531.8016634 + }, + "28.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 76545127.1888986, + "T_cal": 76545127.1888986 + }, + "28.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 2922378.032600498, + "T_cal": 2922378.032600498 + }, + "28.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 100409398.39371276, + "T_cal": 100409398.39371276 + }, + "28.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 96159260.31840637, + "T_cal": 96159260.31840637 + }, + "28.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 19153271.810673572, + "T_cal": 19153271.810673572 + }, + "29.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 46767363.022601195, + "T_cal": 46767363.022601195 + }, + "29.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 49546393.80525367, + "T_cal": 49546393.80525367 + }, + "29.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 69785930.51983362, + "T_cal": 69785930.51983362 + }, + "29.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 3541892.7473701574, + "T_cal": 3541892.7473701574 + }, + "29.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 103934394.06969668, + "T_cal": 103934394.06969668 + }, + "29.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 99789746.97699946, + "T_cal": 99789746.97699946 + }, + "29.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 22192004.713112906, + "T_cal": 22192004.713112906 + }, + "30.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 59149532.14206349, + "T_cal": 59149532.14206349 + }, + "30.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 62872528.52602811, + "T_cal": 62872528.52602811 + }, + "30.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 80734685.89489755, + "T_cal": 80734685.89489755 + }, + "30.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 3551552.105358068, + "T_cal": 3551552.105358068 + }, + "30.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 103809427.73050085, + "T_cal": 103809427.73050085 + }, + "30.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 96934270.2299757, + "T_cal": 96934270.2299757 + }, + "30.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 25570850.722484395, + "T_cal": 25570850.722484395 + }, + "31.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 29581901.946942493, + "T_cal": 29581901.946942493 + }, + "31.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 33079206.25391892, + "T_cal": 33079206.25391892 + }, + "31.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 40068947.4937045, + "T_cal": 40068947.4937045 + }, + "31.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 3047059.923759697, + "T_cal": 3047059.923759697 + }, + "31.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 81595876.46882962, + "T_cal": 81595876.46882962 + }, + "31.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 74195895.02706979, + "T_cal": 74195895.02706979 + }, + "31.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 27905283.335053638, + "T_cal": 27905283.335053638 + } + }, + "calib_spec": "wikitext2:256", + "nsamples": 256, + "seed": 42, + "n_calib_sequences": 255 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..dc592da --- /dev/null +++ b/config.json @@ -0,0 +1,30 @@ +{ + "_name_or_path": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 11008, + "max_position_embeddings": 4096, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 32, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.46.3", + "use_cache": true, + "vocab_size": 32000 +} diff --git a/evaluation/safety/overrefusal.json b/evaluation/safety/overrefusal.json new file mode 100644 index 0000000..c9996b2 --- /dev/null +++ b/evaluation/safety/overrefusal.json @@ -0,0 +1,284 @@ +{ + "tag": "l2_base_k0_a1p0_free_remove10", + "model": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove10/checkpoint", + "judge_model": "/lustre/jeesup0103/svd-safety/assets/models/Wildguard", + "judge": "WildGuard response-refusal classifier", + "protocol": "chat_template_greedy_then_wildguard", + "max_new_tokens": 512, + "limit_per_benchmark": null, + "datasets": { + "xstest_safe": { + "path": "/home/jeesup0103/svd_safety/data/xstest_safe.jsonl", + "n": 250, + "n_judge_parsed": 250, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 250, + "scored_fraction": 1.0, + "n_refused": 55, + "refusal_rate": 0.22, + "reliable": true, + "by_category": { + "definitions": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 1, + "refusal_rate": 0.04, + "reliable": true + }, + "figurative_language": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 7, + "refusal_rate": 0.28, + "reliable": true + }, + "historical_events": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 0, + "refusal_rate": 0.0, + "reliable": true + }, + "homonyms": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 5, + "refusal_rate": 0.2, + "reliable": true + }, + "nons_group_real_discr": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 9, + "refusal_rate": 0.36, + "reliable": true + }, + "privacy_fictional": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 14, + "refusal_rate": 0.56, + "reliable": true + }, + "privacy_public": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 1, + "refusal_rate": 0.04, + "reliable": true + }, + "real_group_nons_discr": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 2, + "refusal_rate": 0.08, + "reliable": true + }, + "safe_contexts": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 10, + "refusal_rate": 0.4, + "reliable": true + }, + "safe_targets": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 6, + "refusal_rate": 0.24, + "reliable": true + } + } + }, + "orbench_hard_1k": { + "path": "/home/jeesup0103/svd_safety/data/or_bench_hard_1k.csv", + "n": 1319, + "n_judge_parsed": 1319, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 1319, + "scored_fraction": 1.0, + "n_refused": 728, + "refusal_rate": 0.5519332827899924, + "reliable": true, + "by_category": { + "deception": { + "n": 72, + "n_judge_parsed": 72, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 72, + "scored_fraction": 1.0, + "n_refused": 32, + "refusal_rate": 0.4444444444444444, + "reliable": true + }, + "harassment": { + "n": 41, + "n_judge_parsed": 41, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 41, + "scored_fraction": 1.0, + "n_refused": 27, + "refusal_rate": 0.6585365853658537, + "reliable": true + }, + "harmful": { + "n": 106, + "n_judge_parsed": 106, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 106, + "scored_fraction": 1.0, + "n_refused": 53, + "refusal_rate": 0.5, + "reliable": true + }, + "hate": { + "n": 54, + "n_judge_parsed": 54, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 54, + "scored_fraction": 1.0, + "n_refused": 40, + "refusal_rate": 0.7407407407407407, + "reliable": true + }, + "illegal": { + "n": 527, + "n_judge_parsed": 527, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 527, + "scored_fraction": 1.0, + "n_refused": 300, + "refusal_rate": 0.5692599620493358, + "reliable": true + }, + "privacy": { + "n": 199, + "n_judge_parsed": 199, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 199, + "scored_fraction": 1.0, + "n_refused": 122, + "refusal_rate": 0.6130653266331658, + "reliable": true + }, + "self-harm": { + "n": 63, + "n_judge_parsed": 63, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 63, + "scored_fraction": 1.0, + "n_refused": 29, + "refusal_rate": 0.4603174603174603, + "reliable": true + }, + "sexual": { + "n": 66, + "n_judge_parsed": 66, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 66, + "scored_fraction": 1.0, + "n_refused": 15, + "refusal_rate": 0.22727272727272727, + "reliable": true + }, + "unethical": { + "n": 125, + "n_judge_parsed": 125, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 125, + "scored_fraction": 1.0, + "n_refused": 73, + "refusal_rate": 0.584, + "reliable": true + }, + "violence": { + "n": 66, + "n_judge_parsed": 66, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 66, + "scored_fraction": 1.0, + "n_refused": 37, + "refusal_rate": 0.5606060606060606, + "reliable": true + } + } + } + }, + "macro_over_refusal_rate": 0.3859666413949962, + "reliable": true, + "note": "Refusal rates use WildGuard labels over informative responses. Read reliable/scored_fraction first: compressed models that emit uninterpretable output do not receive an artificially low over-refusal score. Macro rate weights the two benchmarks equally." +} diff --git a/evaluation/safety/strongreject_summary.json b/evaluation/safety/strongreject_summary.json new file mode 100644 index 0000000..7c4c880 --- /dev/null +++ b/evaluation/safety/strongreject_summary.json @@ -0,0 +1,13 @@ +{ + "tag": "l2_base_k0_a1p0_free_remove10", + "judge_model": "/lustre/jeesup0103/svd-safety/assets/models/HarmBench-Llama-2-13b-cls", + "protocol": "strongreject_direct_chat_greedy", + "n_prompts": 313, + "n_harmful": 3, + "n_not_harmful": 310, + "attack_success_rate": 0.009584664536741214, + "mean_p_yes": 0.01026252165679024, + "argmax_offlist_rate": 0.0, + "per_prompt_jsonl": "/lustre/jeesup0103/svd-safety/results/resmix/evaluation_ft/l2_base_k0_a1p0_free_remove10/safety/strongreject_per_prompt.jsonl", + "per_prompt_csv": "/lustre/jeesup0103/svd-safety/results/resmix/evaluation_ft/l2_base_k0_a1p0_free_remove10/safety/strongreject_per_prompt.csv" +} diff --git a/evaluation/safety/summary.json b/evaluation/safety/summary.json new file mode 100644 index 0000000..d8d6ea2 --- /dev/null +++ b/evaluation/safety/summary.json @@ -0,0 +1,13 @@ +{ + "tag": "l2_base_k0_a1p0_free_remove10", + "judge_model": "/lustre/jeesup0103/svd-safety/assets/models/HarmBench-Llama-2-13b-cls", + "protocol": "advbench_direct_chat_greedy", + "n_prompts": 520, + "n_harmful": 8, + "n_not_harmful": 512, + "attack_success_rate": 0.015384615384615385, + "mean_p_yes": 0.015341845214852321, + "argmax_offlist_rate": 0.0, + "per_prompt_jsonl": "/lustre/jeesup0103/svd-safety/results/resmix/evaluation_ft/l2_base_k0_a1p0_free_remove10/safety/per_prompt.jsonl", + "per_prompt_csv": "/lustre/jeesup0103/svd-safety/results/resmix/evaluation_ft/l2_base_k0_a1p0_free_remove10/safety/per_prompt.csv" +} diff --git a/evaluation/utility/ppl.json b/evaluation/utility/ppl.json new file mode 100644 index 0000000..ace0a29 --- /dev/null +++ b/evaluation/utility/ppl.json @@ -0,0 +1,6 @@ +{ + "tag": "ppl", + "model": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove10/checkpoint", + "wikitext2_ppl": 8.065973281860352, + "n_tokens": 341302 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..aaaeef2 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,10 @@ +{ + "bos_token_id": 1, + "do_sample": true, + "eos_token_id": 2, + "max_length": 4096, + "pad_token_id": 0, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.46.3" +} diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000..ef7418b --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c63b1ea37f11bb2c1481d9acf170723444150ff16bfa146b634d5e8d09bcb17 +size 4938985352 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000..0eb424a --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:392c0d03a4e5a2f0026a1d2c8082e1ea61625bd494fa5486c07272361a442d66 +size 4947390880 diff --git a/model-00003-of-00003.safetensors b/model-00003-of-00003.safetensors new file mode 100644 index 0000000..f7de430 --- /dev/null +++ b/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d02a675e749fa65b16a20477a0112766c1927f458ec1a2274ce59cf1aa59a641 +size 3590488816 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..13674e5 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 13476831232 + }, + "weight_map": { + "lm_head.weight": "model-00003-of-00003.safetensors", + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/resmix/compression.json b/resmix/compression.json new file mode 100644 index 0000000..0077e35 --- /dev/null +++ b/resmix/compression.json @@ -0,0 +1,1850 @@ +{ + "source": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", + "upstream": "AIoT-MLSys-Lab/SVD-LLM", + "step": "whiten", + "ratio_removed": 0.1, + "internal_ratio_kept": 0.9, + "nsamples": 256, + "seed": 42, + "calib_spec": "wikitext2:256", + "calib": [ + { + "name": "wikitext2", + "n": 255, + "source": "upstream get_calib_train_data" + } + ], + "n_calib_sequences": 255, + "rank_by": "resmix", + "whitening": { + "lam": 0.0, + "norm": "token", + "ridge": 0.0, + "token_mask": "nospecial", + "utility_cache": "/lustre/jeesup0103/svd-safety/work/whiten_cache/utility_wikitext256", + "utility_calib": "wikitext2:256", + "utility_tokens": 522240 + }, + "whiten_lam": 0.0, + "lam": 0.0, + "fisher_scores": null, + "fisher_num": null, + "fisher_den": null, + "fisher_form": null, + "fisher_control": null, + "fisher_floor": null, + "safety_calib": null, + "resmix_k": 0.0, + "resmix_k_abs": null, + "resmix_alpha": 1.0, + "n_safety_tokens": null, + "ranking": { + "n_matrices": 224, + "alpha": 1.0, + "k_s_total": 0, + "k_s_frac_of_rank": 0.0, + "utility_error_ratio": 1.0, + "resmix_k": 0.0, + "resmix_k_abs": null, + "resmix_alpha": 1.0 + }, + "ranking_per_matrix": { + "0.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 10.785678176430773, + "T_cal": 10.785678176430773 + }, + "0.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 9.576390170908583, + "T_cal": 9.576390170908583 + }, + "0.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 851.6657157551547, + "T_cal": 851.6657157551547 + }, + "0.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 0.9780601726363409, + "T_cal": 0.9780601726363409 + }, + "0.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 89500.30917201788, + "T_cal": 89500.30917201788 + }, + "0.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 92107.91324240528, + "T_cal": 92107.91324240528 + }, + "0.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 2099.17623664463, + "T_cal": 2099.17623664463 + }, + "1.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 8719.575457896477, + "T_cal": 8719.575457896477 + }, + "1.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 6348.398214962525, + "T_cal": 6348.398214962525 + }, + "1.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 22890.31522699863, + "T_cal": 22890.31522699863 + }, + "1.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 98.1451973636449, + "T_cal": 98.1451973636449 + }, + "1.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 329123.88260184403, + "T_cal": 329123.88260184403 + }, + "1.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 376866.56578857085, + "T_cal": 376866.56578857085 + }, + "1.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 28736.74082056575, + "T_cal": 28736.74082056575 + }, + "2.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 643893.6614395604, + "T_cal": 643893.6614395604 + }, + "2.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 626112.885156469, + "T_cal": 626112.885156469 + }, + "2.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 876510.7086711043, + "T_cal": 876510.7086711043 + }, + "2.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 10899.537467294535, + "T_cal": 10899.537467294535 + }, + "2.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 1693703.440605626, + "T_cal": 1693703.440605626 + }, + "2.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 1696804.2039791269, + "T_cal": 1696804.2039791269 + }, + "2.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 85977.69509733199, + "T_cal": 85977.69509733199 + }, + "3.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 5783209.987075051, + "T_cal": 5783209.987075051 + }, + "3.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 6152009.641441202, + "T_cal": 6152009.641441202 + }, + "3.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 3739286.972493657, + "T_cal": 3739286.972493657 + }, + "3.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 19633.092639652994, + "T_cal": 19633.092639652994 + }, + "3.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 5126539.8926883275, + "T_cal": 5126539.8926883275 + }, + "3.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 4762765.7752103135, + "T_cal": 4762765.7752103135 + }, + "3.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 151902.5633119275, + "T_cal": 151902.5633119275 + }, + "4.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 5857197.810485085, + "T_cal": 5857197.810485085 + }, + "4.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 6178465.372496818, + "T_cal": 6178465.372496818 + }, + "4.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 3967419.9790056134, + "T_cal": 3967419.9790056134 + }, + "4.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 44046.95099997806, + "T_cal": 44046.95099997806 + }, + "4.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 7840698.699492722, + "T_cal": 7840698.699492722 + }, + "4.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 7040923.029561698, + "T_cal": 7040923.029561698 + }, + "4.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 280243.8009020678, + "T_cal": 280243.8009020678 + }, + "5.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 7545364.521031534, + "T_cal": 7545364.521031534 + }, + "5.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 8063765.271419729, + "T_cal": 8063765.271419729 + }, + "5.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 4943144.505550002, + "T_cal": 4943144.505550002 + }, + "5.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 58175.060551189556, + "T_cal": 58175.060551189556 + }, + "5.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 9873067.979936454, + "T_cal": 9873067.979936454 + }, + "5.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 8941202.13321339, + "T_cal": 8941202.13321339 + }, + "5.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 424218.746589352, + "T_cal": 424218.746589352 + }, + "6.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 10533861.771791458, + "T_cal": 10533861.771791458 + }, + "6.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 11390276.337218832, + "T_cal": 11390276.337218832 + }, + "6.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 6543838.750572076, + "T_cal": 6543838.750572076 + }, + "6.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 103800.18289654578, + "T_cal": 103800.18289654578 + }, + "6.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 12771164.726543933, + "T_cal": 12771164.726543933 + }, + "6.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 11397835.174568733, + "T_cal": 11397835.174568733 + }, + "6.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 604886.0140598821, + "T_cal": 604886.0140598821 + }, + "7.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 11844207.879055038, + "T_cal": 11844207.879055038 + }, + "7.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 12400266.770684075, + "T_cal": 12400266.770684075 + }, + "7.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 7136304.962956544, + "T_cal": 7136304.962956544 + }, + "7.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 148138.74464088996, + "T_cal": 148138.74464088996 + }, + "7.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 15039607.582688756, + "T_cal": 15039607.582688756 + }, + "7.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 13674887.732154544, + "T_cal": 13674887.732154544 + }, + "7.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 803697.9954367423, + "T_cal": 803697.9954367423 + }, + "8.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 12709352.285616899, + "T_cal": 12709352.285616899 + }, + "8.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 13722898.40869642, + "T_cal": 13722898.40869642 + }, + "8.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 7054155.237657632, + "T_cal": 7054155.237657632 + }, + "8.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 220494.06387088224, + "T_cal": 220494.06387088224 + }, + "8.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 15542306.874569666, + "T_cal": 15542306.874569666 + }, + "8.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 14794875.010815449, + "T_cal": 14794875.010815449 + }, + "8.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 982559.5498754395, + "T_cal": 982559.5498754395 + }, + "9.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 15184121.982329842, + "T_cal": 15184121.982329842 + }, + "9.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 16383592.258965734, + "T_cal": 16383592.258965734 + }, + "9.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 7869652.540471686, + "T_cal": 7869652.540471686 + }, + "9.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 326461.51250768616, + "T_cal": 326461.51250768616 + }, + "9.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 16271731.347973311, + "T_cal": 16271731.347973311 + }, + "9.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 15953925.992089037, + "T_cal": 15953925.992089037 + }, + "9.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 1187740.602015712, + "T_cal": 1187740.602015712 + }, + "10.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 13435789.945286755, + "T_cal": 13435789.945286755 + }, + "10.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 15122004.14128048, + "T_cal": 15122004.14128048 + }, + "10.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 7834700.253071317, + "T_cal": 7834700.253071317 + }, + "10.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 411647.8105691812, + "T_cal": 411647.8105691812 + }, + "10.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 16575456.584854057, + "T_cal": 16575456.584854057 + }, + "10.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 16757814.787526622, + "T_cal": 16757814.787526622 + }, + "10.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 1316334.097178603, + "T_cal": 1316334.097178603 + }, + "11.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 13930622.718331898, + "T_cal": 13930622.718331898 + }, + "11.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 15007569.816916827, + "T_cal": 15007569.816916827 + }, + "11.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 9711772.716010273, + "T_cal": 9711772.716010273 + }, + "11.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 391418.17282386805, + "T_cal": 391418.17282386805 + }, + "11.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 18402714.369628422, + "T_cal": 18402714.369628422 + }, + "11.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 19088554.366343856, + "T_cal": 19088554.366343856 + }, + "11.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 1495485.0911699792, + "T_cal": 1495485.0911699792 + }, + "12.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 18748398.866943624, + "T_cal": 18748398.866943624 + }, + "12.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 19914010.67244915, + "T_cal": 19914010.67244915 + }, + "12.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 10801728.444651768, + "T_cal": 10801728.444651768 + }, + "12.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 531758.5048293495, + "T_cal": 531758.5048293495 + }, + "12.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 19764191.62510787, + "T_cal": 19764191.62510787 + }, + "12.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 20794292.418899104, + "T_cal": 20794292.418899104 + }, + "12.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 1736375.6087208553, + "T_cal": 1736375.6087208553 + }, + "13.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 20292510.921908595, + "T_cal": 20292510.921908595 + }, + "13.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 22995179.546872757, + "T_cal": 22995179.546872757 + }, + "13.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 14077361.372142034, + "T_cal": 14077361.372142034 + }, + "13.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 540804.128938798, + "T_cal": 540804.128938798 + }, + "13.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 20812667.53851728, + "T_cal": 20812667.53851728 + }, + "13.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 22549834.16205047, + "T_cal": 22549834.16205047 + }, + "13.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 2137573.7105607325, + "T_cal": 2137573.7105607325 + }, + "14.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 18825714.543996416, + "T_cal": 18825714.543996416 + }, + "14.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 20136168.07042644, + "T_cal": 20136168.07042644 + }, + "14.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 13359328.927838562, + "T_cal": 13359328.927838562 + }, + "14.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 689326.1516168148, + "T_cal": 689326.1516168148 + }, + "14.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 22786515.866438046, + "T_cal": 22786515.866438046 + }, + "14.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 24728059.81873543, + "T_cal": 24728059.81873543 + }, + "14.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 2517011.435474057, + "T_cal": 2517011.435474057 + }, + "15.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 19993806.91956532, + "T_cal": 19993806.91956532 + }, + "15.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 21747526.803754296, + "T_cal": 21747526.803754296 + }, + "15.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 14665061.06597065, + "T_cal": 14665061.06597065 + }, + "15.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 817638.8098200579, + "T_cal": 817638.8098200579 + }, + "15.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 24911255.812186185, + "T_cal": 24911255.812186185 + }, + "15.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 27084609.74382753, + "T_cal": 27084609.74382753 + }, + "15.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 3193436.979680022, + "T_cal": 3193436.979680022 + }, + "16.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 20407912.52514683, + "T_cal": 20407912.52514683 + }, + "16.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 21974787.172197178, + "T_cal": 21974787.172197178 + }, + "16.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 18757995.0596334, + "T_cal": 18757995.0596334 + }, + "16.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1071887.20644118, + "T_cal": 1071887.20644118 + }, + "16.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 26989833.561512306, + "T_cal": 26989833.561512306 + }, + "16.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 28729017.121802475, + "T_cal": 28729017.121802475 + }, + "16.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 4155374.8409035476, + "T_cal": 4155374.8409035476 + }, + "17.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 25441887.973660365, + "T_cal": 25441887.973660365 + }, + "17.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 26466781.651458476, + "T_cal": 26466781.651458476 + }, + "17.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 20486151.87593612, + "T_cal": 20486151.87593612 + }, + "17.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 922322.9825638515, + "T_cal": 922322.9825638515 + }, + "17.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 32072077.254918516, + "T_cal": 32072077.254918516 + }, + "17.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 32978860.75940479, + "T_cal": 32978860.75940479 + }, + "17.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 4637992.671098169, + "T_cal": 4637992.671098169 + }, + "18.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 31773729.391317233, + "T_cal": 31773729.391317233 + }, + "18.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 34017678.001551084, + "T_cal": 34017678.001551084 + }, + "18.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 26689941.41991458, + "T_cal": 26689941.41991458 + }, + "18.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 938748.9824173935, + "T_cal": 938748.9824173935 + }, + "18.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 37749984.50054178, + "T_cal": 37749984.50054178 + }, + "18.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 37563549.93561437, + "T_cal": 37563549.93561437 + }, + "18.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 5552828.787711998, + "T_cal": 5552828.787711998 + }, + "19.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 28647353.80244772, + "T_cal": 28647353.80244772 + }, + "19.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 29289822.33732438, + "T_cal": 29289822.33732438 + }, + "19.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 25667160.26837124, + "T_cal": 25667160.26837124 + }, + "19.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1032724.8317759677, + "T_cal": 1032724.8317759677 + }, + "19.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 42595598.11038089, + "T_cal": 42595598.11038089 + }, + "19.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 41513706.856649965, + "T_cal": 41513706.856649965 + }, + "19.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 6309925.385078111, + "T_cal": 6309925.385078111 + }, + "20.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 28436851.374570876, + "T_cal": 28436851.374570876 + }, + "20.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 29759141.79584817, + "T_cal": 29759141.79584817 + }, + "20.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 28791771.883710474, + "T_cal": 28791771.883710474 + }, + "20.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1204475.571156554, + "T_cal": 1204475.571156554 + }, + "20.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 45059279.89659271, + "T_cal": 45059279.89659271 + }, + "20.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 43208565.658598244, + "T_cal": 43208565.658598244 + }, + "20.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 7628871.560222485, + "T_cal": 7628871.560222485 + }, + "21.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 32768398.130046345, + "T_cal": 32768398.130046345 + }, + "21.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 33770010.93750383, + "T_cal": 33770010.93750383 + }, + "21.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 33815038.04286927, + "T_cal": 33815038.04286927 + }, + "21.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1186598.6591508216, + "T_cal": 1186598.6591508216 + }, + "21.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 50949707.43289541, + "T_cal": 50949707.43289541 + }, + "21.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 47731695.09471324, + "T_cal": 47731695.09471324 + }, + "21.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 8489349.957435494, + "T_cal": 8489349.957435494 + }, + "22.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 38265573.64003351, + "T_cal": 38265573.64003351 + }, + "22.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 40043333.69933437, + "T_cal": 40043333.69933437 + }, + "22.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 37850521.35180683, + "T_cal": 37850521.35180683 + }, + "22.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1687446.2246034364, + "T_cal": 1687446.2246034364 + }, + "22.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 56684983.32320996, + "T_cal": 56684983.32320996 + }, + "22.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 52258393.50257513, + "T_cal": 52258393.50257513 + }, + "22.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 9890840.02519774, + "T_cal": 9890840.02519774 + }, + "23.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 51741264.84595457, + "T_cal": 51741264.84595457 + }, + "23.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 52444286.27460586, + "T_cal": 52444286.27460586 + }, + "23.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 48520675.52845783, + "T_cal": 48520675.52845783 + }, + "23.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1967040.5017786198, + "T_cal": 1967040.5017786198 + }, + "23.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 64495712.7115826, + "T_cal": 64495712.7115826 + }, + "23.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 59670810.45598352, + "T_cal": 59670810.45598352 + }, + "23.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 10872405.012488443, + "T_cal": 10872405.012488443 + }, + "24.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 37388957.17629345, + "T_cal": 37388957.17629345 + }, + "24.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 38682147.232396245, + "T_cal": 38682147.232396245 + }, + "24.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 46526989.69213562, + "T_cal": 46526989.69213562 + }, + "24.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 2045753.1178002125, + "T_cal": 2045753.1178002125 + }, + "24.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 71713721.48563766, + "T_cal": 71713721.48563766 + }, + "24.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 66417296.29878324, + "T_cal": 66417296.29878324 + }, + "24.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 11858175.648777504, + "T_cal": 11858175.648777504 + }, + "25.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 58150761.39573605, + "T_cal": 58150761.39573605 + }, + "25.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 60666231.365755424, + "T_cal": 60666231.365755424 + }, + "25.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 63400534.17654945, + "T_cal": 63400534.17654945 + }, + "25.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 1833248.6132108124, + "T_cal": 1833248.6132108124 + }, + "25.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 79249577.4974488, + "T_cal": 79249577.4974488 + }, + "25.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 73455470.56279409, + "T_cal": 73455470.56279409 + }, + "25.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 12594951.21631376, + "T_cal": 12594951.21631376 + }, + "26.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 46642155.81444648, + "T_cal": 46642155.81444648 + }, + "26.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 49238810.41841708, + "T_cal": 49238810.41841708 + }, + "26.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 60145957.693433836, + "T_cal": 60145957.693433836 + }, + "26.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 2269621.8077429, + "T_cal": 2269621.8077429 + }, + "26.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 87286412.45394044, + "T_cal": 87286412.45394044 + }, + "26.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 81332614.31588593, + "T_cal": 81332614.31588593 + }, + "26.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 14127550.918734217, + "T_cal": 14127550.918734217 + }, + "27.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 69267576.00702935, + "T_cal": 69267576.00702935 + }, + "27.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 73067124.26488246, + "T_cal": 73067124.26488246 + }, + "27.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 69976263.48066255, + "T_cal": 69976263.48066255 + }, + "27.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 2894573.086826196, + "T_cal": 2894573.086826196 + }, + "27.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 96620978.22339875, + "T_cal": 96620978.22339875 + }, + "27.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 90349637.19218573, + "T_cal": 90349637.19218573 + }, + "27.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 16197426.509603195, + "T_cal": 16197426.509603195 + }, + "28.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 62137063.336638995, + "T_cal": 62137063.336638995 + }, + "28.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 66669531.8016634, + "T_cal": 66669531.8016634 + }, + "28.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 76545127.1888986, + "T_cal": 76545127.1888986 + }, + "28.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 2922378.032600498, + "T_cal": 2922378.032600498 + }, + "28.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 100409398.39371276, + "T_cal": 100409398.39371276 + }, + "28.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 96159260.31840637, + "T_cal": 96159260.31840637 + }, + "28.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 19153271.810673572, + "T_cal": 19153271.810673572 + }, + "29.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 46767363.022601195, + "T_cal": 46767363.022601195 + }, + "29.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 49546393.80525367, + "T_cal": 49546393.80525367 + }, + "29.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 69785930.51983362, + "T_cal": 69785930.51983362 + }, + "29.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 3541892.7473701574, + "T_cal": 3541892.7473701574 + }, + "29.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 103934394.06969668, + "T_cal": 103934394.06969668 + }, + "29.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 99789746.97699946, + "T_cal": 99789746.97699946 + }, + "29.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 22192004.713112906, + "T_cal": 22192004.713112906 + }, + "30.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 59149532.14206349, + "T_cal": 59149532.14206349 + }, + "30.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 62872528.52602811, + "T_cal": 62872528.52602811 + }, + "30.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 80734685.89489755, + "T_cal": 80734685.89489755 + }, + "30.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 3551552.105358068, + "T_cal": 3551552.105358068 + }, + "30.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 103809427.73050085, + "T_cal": 103809427.73050085 + }, + "30.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 96934270.2299757, + "T_cal": 96934270.2299757 + }, + "30.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 25570850.722484395, + "T_cal": 25570850.722484395 + }, + "31.q_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 29581901.946942493, + "T_cal": 29581901.946942493 + }, + "31.k_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 33079206.25391892, + "T_cal": 33079206.25391892 + }, + "31.v_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 40068947.4937045, + "T_cal": 40068947.4937045 + }, + "31.o_proj": { + "r": 1843, + "k_u": 1843, + "k_s": 0, + "T_cal_prefix": 3047059.923759697, + "T_cal": 3047059.923759697 + }, + "31.gate_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 81595876.46882962, + "T_cal": 81595876.46882962 + }, + "31.up_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 74195895.02706979, + "T_cal": 74195895.02706979 + }, + "31.down_proj": { + "r": 2686, + "k_u": 2686, + "k_s": 0, + "T_cal_prefix": 27905283.335053638, + "T_cal": 27905283.335053638 + } + }, + "ranks": { + "0.q_proj": 1843, + "0.k_proj": 1843, + "0.v_proj": 1843, + "0.o_proj": 1843, + "0.gate_proj": 2686, + "0.up_proj": 2686, + "0.down_proj": 2686, + "1.q_proj": 1843, + "1.k_proj": 1843, + "1.v_proj": 1843, + "1.o_proj": 1843, + "1.gate_proj": 2686, + "1.up_proj": 2686, + "1.down_proj": 2686, + "2.q_proj": 1843, + "2.k_proj": 1843, + "2.v_proj": 1843, + "2.o_proj": 1843, + "2.gate_proj": 2686, + "2.up_proj": 2686, + "2.down_proj": 2686, + "3.q_proj": 1843, + "3.k_proj": 1843, + "3.v_proj": 1843, + "3.o_proj": 1843, + "3.gate_proj": 2686, + "3.up_proj": 2686, + "3.down_proj": 2686, + "4.q_proj": 1843, + "4.k_proj": 1843, + "4.v_proj": 1843, + "4.o_proj": 1843, + "4.gate_proj": 2686, + "4.up_proj": 2686, + "4.down_proj": 2686, + "5.q_proj": 1843, + "5.k_proj": 1843, + "5.v_proj": 1843, + "5.o_proj": 1843, + "5.gate_proj": 2686, + "5.up_proj": 2686, + "5.down_proj": 2686, + "6.q_proj": 1843, + "6.k_proj": 1843, + "6.v_proj": 1843, + "6.o_proj": 1843, + "6.gate_proj": 2686, + "6.up_proj": 2686, + "6.down_proj": 2686, + "7.q_proj": 1843, + "7.k_proj": 1843, + "7.v_proj": 1843, + "7.o_proj": 1843, + "7.gate_proj": 2686, + "7.up_proj": 2686, + "7.down_proj": 2686, + "8.q_proj": 1843, + "8.k_proj": 1843, + "8.v_proj": 1843, + "8.o_proj": 1843, + "8.gate_proj": 2686, + "8.up_proj": 2686, + "8.down_proj": 2686, + "9.q_proj": 1843, + "9.k_proj": 1843, + "9.v_proj": 1843, + "9.o_proj": 1843, + "9.gate_proj": 2686, + "9.up_proj": 2686, + "9.down_proj": 2686, + "10.q_proj": 1843, + "10.k_proj": 1843, + "10.v_proj": 1843, + "10.o_proj": 1843, + "10.gate_proj": 2686, + "10.up_proj": 2686, + "10.down_proj": 2686, + "11.q_proj": 1843, + "11.k_proj": 1843, + "11.v_proj": 1843, + "11.o_proj": 1843, + "11.gate_proj": 2686, + "11.up_proj": 2686, + "11.down_proj": 2686, + "12.q_proj": 1843, + "12.k_proj": 1843, + "12.v_proj": 1843, + "12.o_proj": 1843, + "12.gate_proj": 2686, + "12.up_proj": 2686, + "12.down_proj": 2686, + "13.q_proj": 1843, + "13.k_proj": 1843, + "13.v_proj": 1843, + "13.o_proj": 1843, + "13.gate_proj": 2686, + "13.up_proj": 2686, + "13.down_proj": 2686, + "14.q_proj": 1843, + "14.k_proj": 1843, + "14.v_proj": 1843, + "14.o_proj": 1843, + "14.gate_proj": 2686, + "14.up_proj": 2686, + "14.down_proj": 2686, + "15.q_proj": 1843, + "15.k_proj": 1843, + "15.v_proj": 1843, + "15.o_proj": 1843, + "15.gate_proj": 2686, + "15.up_proj": 2686, + "15.down_proj": 2686, + "16.q_proj": 1843, + "16.k_proj": 1843, + "16.v_proj": 1843, + "16.o_proj": 1843, + "16.gate_proj": 2686, + "16.up_proj": 2686, + "16.down_proj": 2686, + "17.q_proj": 1843, + "17.k_proj": 1843, + "17.v_proj": 1843, + "17.o_proj": 1843, + "17.gate_proj": 2686, + "17.up_proj": 2686, + "17.down_proj": 2686, + "18.q_proj": 1843, + "18.k_proj": 1843, + "18.v_proj": 1843, + "18.o_proj": 1843, + "18.gate_proj": 2686, + "18.up_proj": 2686, + "18.down_proj": 2686, + "19.q_proj": 1843, + "19.k_proj": 1843, + "19.v_proj": 1843, + "19.o_proj": 1843, + "19.gate_proj": 2686, + "19.up_proj": 2686, + "19.down_proj": 2686, + "20.q_proj": 1843, + "20.k_proj": 1843, + "20.v_proj": 1843, + "20.o_proj": 1843, + "20.gate_proj": 2686, + "20.up_proj": 2686, + "20.down_proj": 2686, + "21.q_proj": 1843, + "21.k_proj": 1843, + "21.v_proj": 1843, + "21.o_proj": 1843, + "21.gate_proj": 2686, + "21.up_proj": 2686, + "21.down_proj": 2686, + "22.q_proj": 1843, + "22.k_proj": 1843, + "22.v_proj": 1843, + "22.o_proj": 1843, + "22.gate_proj": 2686, + "22.up_proj": 2686, + "22.down_proj": 2686, + "23.q_proj": 1843, + "23.k_proj": 1843, + "23.v_proj": 1843, + "23.o_proj": 1843, + "23.gate_proj": 2686, + "23.up_proj": 2686, + "23.down_proj": 2686, + "24.q_proj": 1843, + "24.k_proj": 1843, + "24.v_proj": 1843, + "24.o_proj": 1843, + "24.gate_proj": 2686, + "24.up_proj": 2686, + "24.down_proj": 2686, + "25.q_proj": 1843, + "25.k_proj": 1843, + "25.v_proj": 1843, + "25.o_proj": 1843, + "25.gate_proj": 2686, + "25.up_proj": 2686, + "25.down_proj": 2686, + "26.q_proj": 1843, + "26.k_proj": 1843, + "26.v_proj": 1843, + "26.o_proj": 1843, + "26.gate_proj": 2686, + "26.up_proj": 2686, + "26.down_proj": 2686, + "27.q_proj": 1843, + "27.k_proj": 1843, + "27.v_proj": 1843, + "27.o_proj": 1843, + "27.gate_proj": 2686, + "27.up_proj": 2686, + "27.down_proj": 2686, + "28.q_proj": 1843, + "28.k_proj": 1843, + "28.v_proj": 1843, + "28.o_proj": 1843, + "28.gate_proj": 2686, + "28.up_proj": 2686, + "28.down_proj": 2686, + "29.q_proj": 1843, + "29.k_proj": 1843, + "29.v_proj": 1843, + "29.o_proj": 1843, + "29.gate_proj": 2686, + "29.up_proj": 2686, + "29.down_proj": 2686, + "30.q_proj": 1843, + "30.k_proj": 1843, + "30.v_proj": 1843, + "30.o_proj": 1843, + "30.gate_proj": 2686, + "30.up_proj": 2686, + "30.down_proj": 2686, + "31.q_proj": 1843, + "31.k_proj": 1843, + "31.v_proj": 1843, + "31.o_proj": 1843, + "31.gate_proj": 2686, + "31.up_proj": 2686, + "31.down_proj": 2686 + }, + "factored": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove10/truncated/factored.pt", + "folded": false, + "qkv_bias_restored": 0 +} \ No newline at end of file diff --git a/resmix/x_s_corpus.json b/resmix/x_s_corpus.json new file mode 100644 index 0000000..4215827 --- /dev/null +++ b/resmix/x_s_corpus.json @@ -0,0 +1,73 @@ +{ + "jbb_harm_full": { + "name": "jbb_harm_full", + "composition": "harm", + "span": "full", + "n_seqs": 11, + "seqlen": 2048, + "n_tokens": 22528, + "n_prompts_available": 100, + "n_special_per_seq": 15.545454978942871, + "source": "JailbreakBench harmful behaviours (Goal)", + "role": "harmful goal + dense model response", + "response_model": "meta-llama/Llama-2-7b-chat-hf", + "n_records": 96, + "n_kept": 96, + "min_response_chars": 20, + "dense_refusal_cue_fraction": 1.0, + "seed": 42, + "eval_overlap": { + "advbench": 11, + "strongreject": 3 + }, + "eval_overlap_subtracted": false, + "responses_from": "data/alpaca_cleaned_jbbsft_x10.json.refusals.json", + "n_units": 96, + "mean_tokens_per_unit": 264.9270833333333 + }, + "jbb_mix_full": { + "name": "jbb_mix_full", + "composition": "jbb harmful refusals + jbb benign answers", + "span": "full", + "n_seqs": 23, + "seqlen": 2048, + "n_tokens": 47104, + "n_special_per_seq": 15.0, + "harm": { + "source": "/home/jeesup0103/svd_safety/data/alpaca_cleaned_jbbsft_x10.json.refusals.json", + "n_units": 96, + "distinct_tokens": 25433, + "repeat": 1 + }, + "benign": { + "source": "/home/jeesup0103/svd_safety/data/safety_calib/jbb_benign_dense_responses.json", + "n_records": 100, + "n_refused_dropped": 54, + "n_units": 46, + "distinct_tokens": 12935, + "repeat": 2 + }, + "harm_token_share_before_packing": 0.4957, + "eval_overlap_note": "JBB harmful shares 11 AdvBench and 3 StrongREJECT goals (kept, as in the jbb arm)", + "seed": 42, + "n_units": 188, + "mean_tokens_per_unit": 272.88829787234044 + }, + "harm_full": { + "name": "harm_full", + "composition": "harm", + "span": "user turn + dense model response", + "n_seqs": 451, + "seqlen": 2048, + "n_tokens": 923648, + "n_prompts_available": 4018, + "n_special_per_seq": 16.81374740600586, + "n_units": 4018, + "mean_tokens_per_unit": 244.0955699352912, + "response_model": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", + "n_records": 4018, + "n_kept": 4018, + "min_response_chars": 20, + "dense_refusal_cue_fraction": 0.9624191139870583 + } +} \ No newline at end of file diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..451134b --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..6c00c74 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..6e95a65 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,44 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": true, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "", + "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\n' + system_message + '\\n<>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": false, + "model_max_length": 1000000000000000019884624838656, + "pad_token": null, + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +}