commit 7c55e8b4f9cdb7e54d0deb92d03517e15ef1382e Author: ModelHub XC Date: Thu Oct 22 14:01:21 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Jeesup/svd-safety-l2_base_k0_a1p0_free_remove20 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..51089e2 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,126 @@ +LLAMA 2 COMMUNITY LICENSE AGREEMENT +Llama 2 Version Release Date: July 18, 2023 + +"Agreement" means the terms and conditions for use, reproduction, distribution and +modification of the Llama Materials set forth herein. + +"Documentation" means the specifications, manuals and documentation +accompanying Llama 2 distributed by Meta at ai.meta.com/resources/models-and- +libraries/llama-downloads/. + +"Licensee" or "you" means you, or your employer or any other person or entity (if +you are entering into this Agreement on such person or entity's behalf), of the age +required under applicable laws, rules or regulations to provide legal consent and that +has legal authority to bind your employer or such other person or entity if you are +entering in this Agreement on their behalf. + +"Llama 2" means the foundational large language models and software and +algorithms, including machine-learning model code, trained model weights, +inference-enabling code, training-enabling code, fine-tuning enabling code and other +elements of the foregoing distributed by Meta at ai.meta.com/resources/models-and- +libraries/llama-downloads/. + +"Llama Materials" means, collectively, Meta's proprietary Llama 2 and +Documentation (and any portion thereof) made available under this Agreement. + +"Meta" or "we" means Meta Platforms Ireland Limited (if you are located in or, if you +are an entity, your principal place of business is in the EEA or Switzerland) and Meta +Platforms, Inc. (if you are located outside of the EEA or Switzerland). + +By clicking "I Accept" below or by using or distributing any portion or element of the +Llama Materials, you agree to be bound by this Agreement. + +1. License Rights and Redistribution. + + a. Grant of Rights. You are granted a non-exclusive, worldwide, non- +transferable and royalty-free limited license under Meta's intellectual property or +other rights owned by Meta embodied in the Llama Materials to use, reproduce, +distribute, copy, create derivative works of, and make modifications to the Llama +Materials. + + b. Redistribution and Use. + + i. If you distribute or make the Llama Materials, or any derivative works +thereof, available to a third party, you shall provide a copy of this Agreement to such +third party. + ii. If you receive Llama Materials, or any derivative works thereof, from +a Licensee as part of an integrated end user product, then Section 2 of this +Agreement will not apply to you. + + iii. You must retain in all copies of the Llama Materials that you +distribute the following attribution notice within a "Notice" text file distributed as a +part of such copies: "Llama 2 is licensed under the LLAMA 2 Community License, +Copyright (c) Meta Platforms, Inc. All Rights Reserved." + + iv. Your use of the Llama Materials must comply with applicable laws +and regulations (including trade compliance laws and regulations) and adhere to the +Acceptable Use Policy for the Llama Materials (available at +https://ai.meta.com/llama/use-policy), which is hereby incorporated by reference into +this Agreement. + + v. You will not use the Llama Materials or any output or results of the +Llama Materials to improve any other large language model (excluding Llama 2 or +derivative works thereof). + +2. Additional Commercial Terms. If, on the Llama 2 version release date, the +monthly active users of the products or services made available by or for Licensee, +or Licensee's affiliates, is greater than 700 million monthly active users in the +preceding calendar month, you must request a license from Meta, which Meta may +grant to you in its sole discretion, and you are not authorized to exercise any of the +rights under this Agreement unless or until Meta otherwise expressly grants you +such rights. + +3. Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE +LLAMA MATERIALS AND ANY OUTPUT AND RESULTS THEREFROM ARE +PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY +WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR +FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE +FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING +THE LLAMA MATERIALS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR +USE OF THE LLAMA MATERIALS AND ANY OUTPUT AND RESULTS. + +4. Limitation of Liability. IN NO EVENT WILL META OR ITS AFFILIATES BE +LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, +NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS +AGREEMENT, FOR ANY LOST PROFITS OR ANY INDIRECT, SPECIAL, +CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN +IF META OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF +ANY OF THE FOREGOING. + +5. Intellectual Property. + + a. No trademark licenses are granted under this Agreement, and in +connection with the Llama Materials, neither Meta nor Licensee may use any name +or mark owned by or associated with the other or any of its affiliates, except as +required for reasonable and customary use in describing and redistributing the +Llama Materials. + + b. Subject to Meta's ownership of Llama Materials and derivatives made by or +for Meta, with respect to any derivative works and modifications of the Llama +Materials that are made by you, as between you and Meta, you are and will be the +owner of such derivative works and modifications. + + c. If you institute litigation or other proceedings against Meta or any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the Llama +Materials or Llama 2 outputs or results, or any portion of any of the foregoing, +constitutes infringement of intellectual property or other rights owned or licensable +by you, then any licenses granted to you under this Agreement shall terminate as of +the date such litigation or claim is filed or instituted. You will indemnify and hold +harmless Meta from and against any claim by any third party arising out of or related +to your use or distribution of the Llama Materials. + +6. Term and Termination. The term of this Agreement will commence upon your +acceptance of this Agreement or access to the Llama Materials and will continue in +full force and effect until terminated in accordance with the terms and conditions +herein. Meta may terminate this Agreement if you are in breach of any term or +condition of this Agreement. Upon termination of this Agreement, you shall delete +and cease use of the Llama Materials. Sections 3, 4 and 7 shall survive the +termination of this Agreement. + +7. Governing Law and Jurisdiction. This Agreement will be governed and +construed under the laws of the State of California without regard to choice of law +principles, and the UN Convention on Contracts for the International Sale of Goods +does not apply to this Agreement. The courts of California shall have exclusive +jurisdiction of any dispute arising out of this Agreement. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..01a4751 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +license: llama2 +base_model: meta-llama/Llama-2-7b-chat-hf +library_name: transformers +pipeline_tag: text-generation +tags: +- llama2 +- svd +- compression +- safety +- interpretability +--- + +# svd-safety-l2_base_k0_a1p0_free_remove20 + +A Llama-2-7b-chat checkpoint compressed with SVD-LLM to **80.0% of dense +parameters**, then given a **0.0% parameter budget** of restored SVD +components selected by the **`unknown`** rule. + +This is a research artifact from a study of how SVD compression damages safety +behaviour and which component-selection rule best repairs it. It is one cell of a +grid over selection rules and budgets; it is **not** a general-purpose chat model. + +## Provenance + +| field | value | +|---|---| +| base (uncompressed) | `meta-llama/Llama-2-7b-chat-hf` | +| compression | SVD-LLM, 20.00% of parameters removed | +| selection rule | `unknown` | +| restore budget | 0.000% of dense parameters | +| components restored | 0 | +| components swapped out | 0 | +| resulting parameter fraction | 0.7999 | +| seed | 42 | + +## Measured + +| metric | value | +|---|---| +| AdvBench ASR (HarmBench judge) | 0.0135 | +| StrongREJECT ASR (HarmBench judge) | 0.0160 | +| Macro over-refusal (WildGuard) | 0.3860 | +| WikiText-2 perplexity | 8.7680 | + + +## Intended use and limitations + +This checkpoint exists to measure safety/utility trade-offs under compression. +Several arms in the grid are **deliberately safety-degraded** relative to +Llama-2-7b-chat: compression alone raises attack-success rate, and the point of +the study is to quantify that and test recovery. Treat any given cell as an +experimental subject, not as a deployable assistant, and evaluate it yourself +before drawing conclusions from it. + +## Licence + +Llama 2 Community License. `LICENSE.txt` and `USE_POLICY.md` are included in this repository, and use of this derivative is bound by them. Built with Llama 2. diff --git a/USE_POLICY.md b/USE_POLICY.md new file mode 100644 index 0000000..abbcc19 --- /dev/null +++ b/USE_POLICY.md @@ -0,0 +1,50 @@ +# Llama 2 Acceptable Use Policy + +Meta is committed to promoting safe and fair use of its tools and features, including Llama 2. If you access or use Llama 2, you agree to this Acceptable Use Policy (“Policy”). The most recent copy of this policy can be found at [ai.meta.com/llama/use-policy](http://ai.meta.com/llama/use-policy). + +## Prohibited Uses +We want everyone to use Llama 2 safely and responsibly. You agree you will not use, or allow others to use, Llama 2 to: + +1. Violate the law or others’ rights, including to: + 1. Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content, such as: + 1. Violence or terrorism + 2. Exploitation or harm to children, including the solicitation, creation, acquisition, or dissemination of child exploitative content or failure to report Child Sexual Abuse Material + 3. Human trafficking, exploitation, and sexual violence + 4. The illegal distribution of information or materials to minors, including obscene materials, or failure to employ legally required age-gating in connection with such information or materials. + 5. Sexual solicitation + 6. Any other criminal activity + 2. Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals + 3. Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services + 4. Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices + 5. Collect, process, disclose, generate, or infer health, demographic, or other sensitive personal or private information about individuals without rights and consents required by applicable laws + 6. Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any products or services using the Llama 2 Materials + 7. Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system + + + +2. Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including use of Llama 2 related to the following: + 1. Military, warfare, nuclear industries or applications, espionage, use for materials or activities that are subject to the International Traffic Arms Regulations (ITAR) maintained by the United States Department of State + 2. Guns and illegal weapons (including weapon development) + 3. Illegal drugs and regulated/controlled substances + 4. Operation of critical infrastructure, transportation technologies, or heavy machinery + 5. Self-harm or harm to others, including suicide, cutting, and eating disorders + 6. Any content intended to incite or promote violence, abuse, or any infliction of bodily harm to an individual + + + +3. Intentionally deceive or mislead others, including use of Llama 2 related to the following: + 1. Generating, promoting, or furthering fraud or the creation or promotion of disinformation + 2. Generating, promoting, or furthering defamatory content, including the creation of defamatory statements, images, or other content + 3. Generating, promoting, or further distributing spam + 4. Impersonating another individual without consent, authorization, or legal right + 5. Representing that the use of Llama 2 or outputs are human-generated + 6. Generating or facilitating false online engagement, including fake reviews and other means of fake online engagement +4. Fail to appropriately disclose to end users any known dangers of your AI system + +Please report any violation of this Policy, software “bug,” or other problems that could lead to a violation of this Policy through one of the following means: + +* Reporting issues with the model: [github.com/facebookresearch/llama](http://github.com/facebookresearch/llama) +* Reporting risky content generated by the model: [developers.facebook.com/llama_output_feedback](http://developers.facebook.com/llama_output_feedback) +* Reporting bugs and security concerns: [facebook.com/whitehat/info](http://facebook.com/whitehat/info) +* Reporting violations of the Acceptable Use Policy or unlicensed uses of Llama: [LlamaUseReport@meta.com](mailto:LlamaUseReport@meta.com) + diff --git a/compression.json b/compression.json new file mode 100644 index 0000000..f36c246 --- /dev/null +++ b/compression.json @@ -0,0 +1,2060 @@ +{ + "source": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", + "upstream": "AIoT-MLSys-Lab/SVD-LLM", + "stage": "finetuned", + "ratio_removed": 0.2, + "internal_ratio_kept": 0.8, + "from_pt": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove20/finetune_v/merge.pt", + "biases_verified": 0, + "mean_rel_err": 0.41394592352610615, + "max_rel_err": 1.211578130722046, + "param_fraction": 0.7998957658678757, + "ranks": { + "0.q_proj": 1638, + "0.k_proj": 1638, + "0.v_proj": 1638, + "0.o_proj": 1638, + "0.gate_proj": 2388, + "0.up_proj": 2388, + "0.down_proj": 2388, + "1.q_proj": 1638, + "1.k_proj": 1638, + "1.v_proj": 1638, + "1.o_proj": 1638, + "1.gate_proj": 2388, + "1.up_proj": 2388, + "1.down_proj": 2388, + "2.q_proj": 1638, + "2.k_proj": 1638, + "2.v_proj": 1638, + "2.o_proj": 1638, + "2.gate_proj": 2388, + "2.up_proj": 2388, + "2.down_proj": 2388, + "3.q_proj": 1638, + "3.k_proj": 1638, + "3.v_proj": 1638, + "3.o_proj": 1638, + "3.gate_proj": 2388, + "3.up_proj": 2388, + "3.down_proj": 2388, + "4.q_proj": 1638, + "4.k_proj": 1638, + "4.v_proj": 1638, + "4.o_proj": 1638, + "4.gate_proj": 2388, + "4.up_proj": 2388, + "4.down_proj": 2388, + "5.q_proj": 1638, + "5.k_proj": 1638, + "5.v_proj": 1638, + "5.o_proj": 1638, + "5.gate_proj": 2388, + "5.up_proj": 2388, + "5.down_proj": 2388, + "6.q_proj": 1638, + "6.k_proj": 1638, + "6.v_proj": 1638, + "6.o_proj": 1638, + "6.gate_proj": 2388, + "6.up_proj": 2388, + "6.down_proj": 2388, + "7.q_proj": 1638, + "7.k_proj": 1638, + "7.v_proj": 1638, + "7.o_proj": 1638, + "7.gate_proj": 2388, + "7.up_proj": 2388, + "7.down_proj": 2388, + "8.q_proj": 1638, + "8.k_proj": 1638, + "8.v_proj": 1638, + "8.o_proj": 1638, + "8.gate_proj": 2388, + "8.up_proj": 2388, + "8.down_proj": 2388, + "9.q_proj": 1638, + "9.k_proj": 1638, + "9.v_proj": 1638, + "9.o_proj": 1638, + "9.gate_proj": 2388, + "9.up_proj": 2388, + "9.down_proj": 2388, + "10.q_proj": 1638, + "10.k_proj": 1638, + "10.v_proj": 1638, + "10.o_proj": 1638, + "10.gate_proj": 2388, + "10.up_proj": 2388, + "10.down_proj": 2388, + "11.q_proj": 1638, + "11.k_proj": 1638, + "11.v_proj": 1638, + "11.o_proj": 1638, + "11.gate_proj": 2388, + "11.up_proj": 2388, + "11.down_proj": 2388, + "12.q_proj": 1638, + "12.k_proj": 1638, + "12.v_proj": 1638, + "12.o_proj": 1638, + "12.gate_proj": 2388, + "12.up_proj": 2388, + "12.down_proj": 2388, + "13.q_proj": 1638, + "13.k_proj": 1638, + "13.v_proj": 1638, + "13.o_proj": 1638, + "13.gate_proj": 2388, + "13.up_proj": 2388, + "13.down_proj": 2388, + "14.q_proj": 1638, + "14.k_proj": 1638, + "14.v_proj": 1638, + "14.o_proj": 1638, + "14.gate_proj": 2388, + "14.up_proj": 2388, + "14.down_proj": 2388, + "15.q_proj": 1638, + "15.k_proj": 1638, + "15.v_proj": 1638, + "15.o_proj": 1638, + "15.gate_proj": 2388, + "15.up_proj": 2388, + "15.down_proj": 2388, + "16.q_proj": 1638, + "16.k_proj": 1638, + "16.v_proj": 1638, + "16.o_proj": 1638, + "16.gate_proj": 2388, + "16.up_proj": 2388, + "16.down_proj": 2388, + "17.q_proj": 1638, + "17.k_proj": 1638, + "17.v_proj": 1638, + "17.o_proj": 1638, + "17.gate_proj": 2388, + "17.up_proj": 2388, + "17.down_proj": 2388, + "18.q_proj": 1638, + "18.k_proj": 1638, + "18.v_proj": 1638, + "18.o_proj": 1638, + "18.gate_proj": 2388, + "18.up_proj": 2388, + "18.down_proj": 2388, + "19.q_proj": 1638, + "19.k_proj": 1638, + "19.v_proj": 1638, + "19.o_proj": 1638, + "19.gate_proj": 2388, + "19.up_proj": 2388, + "19.down_proj": 2388, + "20.q_proj": 1638, + "20.k_proj": 1638, + "20.v_proj": 1638, + "20.o_proj": 1638, + "20.gate_proj": 2388, + "20.up_proj": 2388, + "20.down_proj": 2388, + "21.q_proj": 1638, + "21.k_proj": 1638, + "21.v_proj": 1638, + "21.o_proj": 1638, + "21.gate_proj": 2388, + "21.up_proj": 2388, + "21.down_proj": 2388, + "22.q_proj": 1638, + "22.k_proj": 1638, + "22.v_proj": 1638, + "22.o_proj": 1638, + "22.gate_proj": 2388, + "22.up_proj": 2388, + "22.down_proj": 2388, + "23.q_proj": 1638, + "23.k_proj": 1638, + "23.v_proj": 1638, + "23.o_proj": 1638, + "23.gate_proj": 2388, + "23.up_proj": 2388, + "23.down_proj": 2388, + "24.q_proj": 1638, + "24.k_proj": 1638, + "24.v_proj": 1638, + "24.o_proj": 1638, + "24.gate_proj": 2388, + "24.up_proj": 2388, + "24.down_proj": 2388, + "25.q_proj": 1638, + "25.k_proj": 1638, + "25.v_proj": 1638, + "25.o_proj": 1638, + "25.gate_proj": 2388, + "25.up_proj": 2388, + "25.down_proj": 2388, + "26.q_proj": 1638, + "26.k_proj": 1638, + "26.v_proj": 1638, + "26.o_proj": 1638, + "26.gate_proj": 2388, + "26.up_proj": 2388, + "26.down_proj": 2388, + "27.q_proj": 1638, + "27.k_proj": 1638, + "27.v_proj": 1638, + "27.o_proj": 1638, + "27.gate_proj": 2388, + "27.up_proj": 2388, + "27.down_proj": 2388, + "28.q_proj": 1638, + "28.k_proj": 1638, + "28.v_proj": 1638, + "28.o_proj": 1638, + "28.gate_proj": 2388, + "28.up_proj": 2388, + "28.down_proj": 2388, + "29.q_proj": 1638, + "29.k_proj": 1638, + "29.v_proj": 1638, + "29.o_proj": 1638, + "29.gate_proj": 2388, + "29.up_proj": 2388, + "29.down_proj": 2388, + "30.q_proj": 1638, + "30.k_proj": 1638, + "30.v_proj": 1638, + "30.o_proj": 1638, + "30.gate_proj": 2388, + "30.up_proj": 2388, + "30.down_proj": 2388, + "31.q_proj": 1638, + "31.k_proj": 1638, + "31.v_proj": 1638, + "31.o_proj": 1638, + "31.gate_proj": 2388, + "31.up_proj": 2388, + "31.down_proj": 2388 + }, + "rel_err": { + "0.q_proj": 0.20255117118358612, + "0.k_proj": 0.2501266598701477, + "0.v_proj": 0.28070899844169617, + "0.o_proj": 0.27957290410995483, + "0.gate_proj": 0.3386726677417755, + "0.up_proj": 0.3481939435005188, + "0.down_proj": 0.4063560962677002, + "1.q_proj": 0.10419954359531403, + "1.k_proj": 0.08726953715085983, + "1.v_proj": 0.3839426636695862, + "1.o_proj": 0.3093242347240448, + "1.gate_proj": 0.3187064230442047, + "1.up_proj": 0.36167171597480774, + "1.down_proj": 0.4551885426044464, + "2.q_proj": 0.5643028616905212, + "2.k_proj": 1.211578130722046, + "2.v_proj": 0.5239243507385254, + "2.o_proj": 0.4612278342247009, + "2.gate_proj": 0.3821312487125397, + "2.up_proj": 0.41011422872543335, + "2.down_proj": 0.4514543414115906, + "3.q_proj": 0.29918423295021057, + "3.k_proj": 0.29636576771736145, + "3.v_proj": 0.4295484125614166, + "3.o_proj": 0.453740656375885, + "3.gate_proj": 0.4301033020019531, + "3.up_proj": 0.44765758514404297, + "3.down_proj": 0.42106202244758606, + "4.q_proj": 0.29087331891059875, + "4.k_proj": 0.2925068438053131, + "4.v_proj": 0.4270686209201813, + "4.o_proj": 0.4564551115036011, + "4.gate_proj": 0.4318659007549286, + "4.up_proj": 0.454406201839447, + "4.down_proj": 0.42497676610946655, + "5.q_proj": 0.33008909225463867, + "5.k_proj": 0.30816328525543213, + "5.v_proj": 0.4289003014564514, + "5.o_proj": 0.4635000228881836, + "5.gate_proj": 0.42839235067367554, + "5.up_proj": 0.45572370290756226, + "5.down_proj": 0.4169948995113373, + "6.q_proj": 0.2949267625808716, + "6.k_proj": 0.30057793855667114, + "6.v_proj": 0.41563349962234497, + "6.o_proj": 0.4318769872188568, + "6.gate_proj": 0.4237746596336365, + "6.up_proj": 0.45605388283729553, + "6.down_proj": 0.4162188172340393, + "7.q_proj": 0.29803186655044556, + "7.k_proj": 0.3041437864303589, + "7.v_proj": 0.4097481667995453, + "7.o_proj": 0.4478301703929901, + "7.gate_proj": 0.42146944999694824, + "7.up_proj": 0.4560389816761017, + "7.down_proj": 0.4105474352836609, + "8.q_proj": 0.3068524897098541, + "8.k_proj": 0.31753095984458923, + "8.v_proj": 0.39811229705810547, + "8.o_proj": 0.44515687227249146, + "8.gate_proj": 0.4194181561470032, + "8.up_proj": 0.4524802565574646, + "8.down_proj": 0.40698784589767456, + "9.q_proj": 0.3244834840297699, + "9.k_proj": 0.32932180166244507, + "9.v_proj": 0.39580363035202026, + "9.o_proj": 0.4497195780277252, + "9.gate_proj": 0.41637614369392395, + "9.up_proj": 0.44777822494506836, + "9.down_proj": 0.40672504901885986, + "10.q_proj": 0.30439677834510803, + "10.k_proj": 0.3114989697933197, + "10.v_proj": 0.3912244737148285, + "10.o_proj": 0.4084624946117401, + "10.gate_proj": 0.41183236241340637, + "10.up_proj": 0.4439256191253662, + "10.down_proj": 0.40340298414230347, + "11.q_proj": 0.43780505657196045, + "11.k_proj": 0.5263542532920837, + "11.v_proj": 0.40769729018211365, + "11.o_proj": 0.3999696969985962, + "11.gate_proj": 0.41143807768821716, + "11.up_proj": 0.44354668259620667, + "11.down_proj": 0.4089212119579315, + "12.q_proj": 0.31846413016319275, + "12.k_proj": 0.3182394504547119, + "12.v_proj": 0.39351150393486023, + "12.o_proj": 0.4362848401069641, + "12.gate_proj": 0.41502130031585693, + "12.up_proj": 0.4439792037010193, + "12.down_proj": 0.4085458219051361, + "13.q_proj": 0.3248235881328583, + "13.k_proj": 0.3383275270462036, + "13.v_proj": 0.41681525111198425, + "13.o_proj": 0.42809635400772095, + "13.gate_proj": 0.4172954857349396, + "13.up_proj": 0.4466675817966461, + "13.down_proj": 0.4194817543029785, + "14.q_proj": 0.3109501600265503, + "14.k_proj": 0.3148863911628723, + "14.v_proj": 0.4114783704280853, + "14.o_proj": 0.4256104826927185, + "14.gate_proj": 0.4175460636615753, + "14.up_proj": 0.44549453258514404, + "14.down_proj": 0.42101749777793884, + "15.q_proj": 0.3258931636810303, + "15.k_proj": 0.3283395767211914, + "15.v_proj": 0.41744932532310486, + "15.o_proj": 0.4321882724761963, + "15.gate_proj": 0.4215778112411499, + "15.up_proj": 0.4495786130428314, + "15.down_proj": 0.4331757724285126, + "16.q_proj": 0.32395583391189575, + "16.k_proj": 0.32779502868652344, + "16.v_proj": 0.44108811020851135, + "16.o_proj": 0.45973899960517883, + "16.gate_proj": 0.4202415645122528, + "16.up_proj": 0.4471694529056549, + "16.down_proj": 0.4438062310218811, + "17.q_proj": 0.3577421307563782, + "17.k_proj": 0.35680827498435974, + "17.v_proj": 0.45505425333976746, + "17.o_proj": 0.469711035490036, + "17.gate_proj": 0.42273563146591187, + "17.up_proj": 0.4474468529224396, + "17.down_proj": 0.44114768505096436, + "18.q_proj": 0.3783473074436188, + "18.k_proj": 0.3835922181606293, + "18.v_proj": 0.4610622525215149, + "18.o_proj": 0.49814432859420776, + "18.gate_proj": 0.4297594726085663, + "18.up_proj": 0.4531024396419525, + "18.down_proj": 0.4420911371707916, + "19.q_proj": 0.3647320866584778, + "19.k_proj": 0.36293184757232666, + "19.v_proj": 0.44766291975975037, + "19.o_proj": 0.486400842666626, + "19.gate_proj": 0.43388551473617554, + "19.up_proj": 0.45401647686958313, + "19.down_proj": 0.44226446747779846, + "20.q_proj": 0.3568756580352783, + "20.k_proj": 0.35885825753211975, + "20.v_proj": 0.4588818848133087, + "20.o_proj": 0.48604533076286316, + "20.gate_proj": 0.4321610629558563, + "20.up_proj": 0.4511963725090027, + "20.down_proj": 0.4513836205005646, + "21.q_proj": 0.3690851032733917, + "21.k_proj": 0.3707112669944763, + "21.v_proj": 0.45562759041786194, + "21.o_proj": 0.5003960728645325, + "21.gate_proj": 0.43256253004074097, + "21.up_proj": 0.4508614242076874, + "21.down_proj": 0.4476333558559418, + "22.q_proj": 0.3774496018886566, + "22.k_proj": 0.38153403997421265, + "22.v_proj": 0.46478772163391113, + "22.o_proj": 0.496971994638443, + "22.gate_proj": 0.4330741763114929, + "22.up_proj": 0.4499264657497406, + "22.down_proj": 0.4451128840446472, + "23.q_proj": 0.40985897183418274, + "23.k_proj": 0.407537579536438, + "23.v_proj": 0.4638177752494812, + "23.o_proj": 0.4878511130809784, + "23.gate_proj": 0.435762882232666, + "23.up_proj": 0.45139607787132263, + "23.down_proj": 0.4385303258895874, + "24.q_proj": 0.35800909996032715, + "24.k_proj": 0.36233046650886536, + "24.v_proj": 0.45714786648750305, + "24.o_proj": 0.4935578405857086, + "24.gate_proj": 0.4361893832683563, + "24.up_proj": 0.45203301310539246, + "24.down_proj": 0.4328676164150238, + "25.q_proj": 0.404552161693573, + "25.k_proj": 0.4057922959327698, + "25.v_proj": 0.46598517894744873, + "25.o_proj": 0.48773059248924255, + "25.gate_proj": 0.43838492035865784, + "25.up_proj": 0.45330461859703064, + "25.down_proj": 0.4277186989784241, + "26.q_proj": 0.3698907196521759, + "26.k_proj": 0.37772369384765625, + "26.v_proj": 0.45376813411712646, + "26.o_proj": 0.5130496621131897, + "26.gate_proj": 0.44034111499786377, + "26.up_proj": 0.45590564608573914, + "26.down_proj": 0.42648565769195557, + "27.q_proj": 0.4138648509979248, + "27.k_proj": 0.42276251316070557, + "27.v_proj": 0.4709629714488983, + "27.o_proj": 0.488780677318573, + "27.gate_proj": 0.4432637393474579, + "27.up_proj": 0.45742523670196533, + "27.down_proj": 0.426099568605423, + "28.q_proj": 0.3980882167816162, + "28.k_proj": 0.4081520438194275, + "28.v_proj": 0.46830081939697266, + "28.o_proj": 0.4866650402545929, + "28.gate_proj": 0.442861944437027, + "28.up_proj": 0.45651155710220337, + "28.down_proj": 0.4337374269962311, + "29.q_proj": 0.36964160203933716, + "29.k_proj": 0.3767383396625519, + "29.v_proj": 0.4603976607322693, + "29.o_proj": 0.48703300952911377, + "29.gate_proj": 0.44214755296707153, + "29.up_proj": 0.4528387486934662, + "29.down_proj": 0.44032400846481323, + "30.q_proj": 0.3995320498943329, + "30.k_proj": 0.4057026505470276, + "30.v_proj": 0.473531574010849, + "30.o_proj": 0.4958701431751251, + "30.gate_proj": 0.4368852376937866, + "30.up_proj": 0.4417875111103058, + "30.down_proj": 0.44819220900535583, + "31.q_proj": 0.3507238030433655, + "31.k_proj": 0.3568456768989563, + "31.v_proj": 0.45507192611694336, + "31.o_proj": 0.4658190906047821, + "31.gate_proj": 0.4278441369533539, + "31.up_proj": 0.4285564422607422, + "31.down_proj": 0.44620150327682495 + }, + "whitening": { + "lam": 0.0, + "norm": "token", + "ridge": 0.0, + "token_mask": "nospecial", + "utility_cache": "/lustre/jeesup0103/svd-safety/work/whiten_cache/utility_wikitext256", + "utility_calib": "wikitext2:256", + "utility_tokens": 522240 + }, + "rank_by": "resmix", + "resmix_k": 0.0, + "resmix_k_abs": null, + "ranking": { + "n_matrices": 224, + "alpha": 1.0, + "k_s_total": 0, + "k_s_frac_of_rank": 0.0, + "utility_error_ratio": 1.0, + "resmix_k": 0.0, + "resmix_k_abs": null, + "resmix_alpha": 1.0 + }, + "ranking_per_matrix": { + "0.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 20.533438156264545, + "T_cal": 20.533438156264545 + }, + "0.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 18.145091391082453, + "T_cal": 18.145091391082453 + }, + "0.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1367.2978993583235, + "T_cal": 1367.2978993583235 + }, + "0.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1.7652817030071697, + "T_cal": 1.7652817030071697 + }, + "0.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 147005.47186107977, + "T_cal": 147005.47186107977 + }, + "0.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 151270.08798223134, + "T_cal": 151270.08798223134 + }, + "0.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 3251.7172484408165, + "T_cal": 3251.7172484408165 + }, + "1.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 15810.360461051278, + "T_cal": 15810.360461051278 + }, + "1.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 11679.950873318438, + "T_cal": 11679.950873318438 + }, + "1.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 34601.29388128947, + "T_cal": 34601.29388128947 + }, + "1.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 210.1460327002143, + "T_cal": 210.1460327002143 + }, + "1.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 554807.0704461301, + "T_cal": 554807.0704461301 + }, + "1.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 632577.4831049199, + "T_cal": 632577.4831049199 + }, + "1.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 41926.18052799249, + "T_cal": 41926.18052799249 + }, + "2.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1075742.3213626854, + "T_cal": 1075742.3213626854 + }, + "2.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1061928.8193543167, + "T_cal": 1061928.8193543167 + }, + "2.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1257767.9325557249, + "T_cal": 1257767.9325557249 + }, + "2.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 16470.990055876522, + "T_cal": 16470.990055876522 + }, + "2.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 2616354.6153592253, + "T_cal": 2616354.6153592253 + }, + "2.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 2619809.05808633, + "T_cal": 2619809.05808633 + }, + "2.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 122572.30074651254, + "T_cal": 122572.30074651254 + }, + "3.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 8616199.692200687, + "T_cal": 8616199.692200687 + }, + "3.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 9252912.097098961, + "T_cal": 9252912.097098961 + }, + "3.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 5181894.684841514, + "T_cal": 5181894.684841514 + }, + "3.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 28070.226419795454, + "T_cal": 28070.226419795454 + }, + "3.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 7535360.907474672, + "T_cal": 7535360.907474672 + }, + "3.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 6997858.889793897, + "T_cal": 6997858.889793897 + }, + "3.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 216506.34547028563, + "T_cal": 216506.34547028563 + }, + "4.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 8576586.798566049, + "T_cal": 8576586.798566049 + }, + "4.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 9103491.821577858, + "T_cal": 9103491.821577858 + }, + "4.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 5447171.7095543165, + "T_cal": 5447171.7095543165 + }, + "4.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 62961.396595317005, + "T_cal": 62961.396595317005 + }, + "4.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 11368285.653579928, + "T_cal": 11368285.653579928 + }, + "4.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 10192894.986038502, + "T_cal": 10192894.986038502 + }, + "4.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 399140.58106265066, + "T_cal": 399140.58106265066 + }, + "5.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 10904378.784864234, + "T_cal": 10904378.784864234 + }, + "5.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 11716277.222077748, + "T_cal": 11716277.222077748 + }, + "5.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 6791614.363699443, + "T_cal": 6791614.363699443 + }, + "5.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 83191.98936714102, + "T_cal": 83191.98936714102 + }, + "5.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 14261817.455468014, + "T_cal": 14261817.455468014 + }, + "5.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 12898852.177619234, + "T_cal": 12898852.177619234 + }, + "5.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 605437.6144528482, + "T_cal": 605437.6144528482 + }, + "6.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 15022264.490293957, + "T_cal": 15022264.490293957 + }, + "6.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 16281336.734301325, + "T_cal": 16281336.734301325 + }, + "6.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 8978679.766875453, + "T_cal": 8978679.766875453 + }, + "6.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 146576.1915614182, + "T_cal": 146576.1915614182 + }, + "6.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 18379550.492156003, + "T_cal": 18379550.492156003 + }, + "6.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 16360762.366840478, + "T_cal": 16360762.366840478 + }, + "6.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 865823.7044643352, + "T_cal": 865823.7044643352 + }, + "7.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 16961003.97794759, + "T_cal": 16961003.97794759 + }, + "7.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 17778413.16036941, + "T_cal": 17778413.16036941 + }, + "7.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 9833500.891810104, + "T_cal": 9833500.891810104 + }, + "7.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 207621.14171330316, + "T_cal": 207621.14171330316 + }, + "7.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 21548258.973763738, + "T_cal": 21548258.973763738 + }, + "7.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 19538704.244472977, + "T_cal": 19538704.244472977 + }, + "7.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 1154651.4831785636, + "T_cal": 1154651.4831785636 + }, + "8.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 18144793.681941804, + "T_cal": 18144793.681941804 + }, + "8.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 19643662.875985958, + "T_cal": 19643662.875985958 + }, + "8.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 9723547.035088653, + "T_cal": 9723547.035088653 + }, + "8.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 309933.4441164221, + "T_cal": 309933.4441164221 + }, + "8.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 22219645.429822985, + "T_cal": 22219645.429822985 + }, + "8.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 21110504.855032086, + "T_cal": 21110504.855032086 + }, + "8.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 1407579.4474217317, + "T_cal": 1407579.4474217317 + }, + "9.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 21577254.884359382, + "T_cal": 21577254.884359382 + }, + "9.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 23354272.865979496, + "T_cal": 23354272.865979496 + }, + "9.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 10870858.616251059, + "T_cal": 10870858.616251059 + }, + "9.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 460797.9531125524, + "T_cal": 460797.9531125524 + }, + "9.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 23261039.5382552, + "T_cal": 23261039.5382552 + }, + "9.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 22760975.831206925, + "T_cal": 22760975.831206925 + }, + "9.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 1698712.7164256899, + "T_cal": 1698712.7164256899 + }, + "10.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 19142257.364713803, + "T_cal": 19142257.364713803 + }, + "10.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 21642343.533273995, + "T_cal": 21642343.533273995 + }, + "10.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 10805862.494584344, + "T_cal": 10805862.494584344 + }, + "10.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 583564.7478921877, + "T_cal": 583564.7478921877 + }, + "10.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 23722052.52198033, + "T_cal": 23722052.52198033 + }, + "10.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 23921352.30383531, + "T_cal": 23921352.30383531 + }, + "10.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 1885944.1773879488, + "T_cal": 1885944.1773879488 + }, + "11.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 19812212.040702116, + "T_cal": 19812212.040702116 + }, + "11.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 21325013.33195281, + "T_cal": 21325013.33195281 + }, + "11.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 13443340.955584655, + "T_cal": 13443340.955584655 + }, + "11.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 554587.2929435676, + "T_cal": 554587.2929435676 + }, + "11.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 26289978.176920943, + "T_cal": 26289978.176920943 + }, + "11.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 27192678.451939438, + "T_cal": 27192678.451939438 + }, + "11.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 2138458.336182276, + "T_cal": 2138458.336182276 + }, + "12.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 26566691.310539044, + "T_cal": 26566691.310539044 + }, + "12.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 28198272.399621785, + "T_cal": 28198272.399621785 + }, + "12.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 14937439.7554898, + "T_cal": 14937439.7554898 + }, + "12.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 744547.0841763078, + "T_cal": 744547.0841763078 + }, + "12.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 28233288.38660199, + "T_cal": 28233288.38660199 + }, + "12.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 29627056.858521752, + "T_cal": 29627056.858521752 + }, + "12.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 2484347.4290266912, + "T_cal": 2484347.4290266912 + }, + "13.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 28594427.825724967, + "T_cal": 28594427.825724967 + }, + "13.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 32408434.376211423, + "T_cal": 32408434.376211423 + }, + "13.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 19320563.55230657, + "T_cal": 19320563.55230657 + }, + "13.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 760267.6486922672, + "T_cal": 760267.6486922672 + }, + "13.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 29710529.160073094, + "T_cal": 29710529.160073094 + }, + "13.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 32093225.825580716, + "T_cal": 32093225.825580716 + }, + "13.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 3047495.20318145, + "T_cal": 3047495.20318145 + }, + "14.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 26564102.81909692, + "T_cal": 26564102.81909692 + }, + "14.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 28371669.477075536, + "T_cal": 28371669.477075536 + }, + "14.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 18403309.45612433, + "T_cal": 18403309.45612433 + }, + "14.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 961585.4964648974, + "T_cal": 961585.4964648974 + }, + "14.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 32568338.730469096, + "T_cal": 32568338.730469096 + }, + "14.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 35260137.34940484, + "T_cal": 35260137.34940484 + }, + "14.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 3590868.026428486, + "T_cal": 3590868.026428486 + }, + "15.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 28171399.800540484, + "T_cal": 28171399.800540484 + }, + "15.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 30588812.300417174, + "T_cal": 30588812.300417174 + }, + "15.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 20111418.348540258, + "T_cal": 20111418.348540258 + }, + "15.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1140757.0870852754, + "T_cal": 1140757.0870852754 + }, + "15.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 35599225.039749816, + "T_cal": 35599225.039749816 + }, + "15.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 38562927.699108556, + "T_cal": 38562927.699108556 + }, + "15.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 4532126.281073529, + "T_cal": 4532126.281073529 + }, + "16.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 28665105.452431396, + "T_cal": 28665105.452431396 + }, + "16.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 30869692.599226408, + "T_cal": 30869692.599226408 + }, + "16.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 25487438.164353833, + "T_cal": 25487438.164353833 + }, + "16.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1505170.2068931672, + "T_cal": 1505170.2068931672 + }, + "16.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 38665250.473643295, + "T_cal": 38665250.473643295 + }, + "16.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 41007811.453693196, + "T_cal": 41007811.453693196 + }, + "16.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 5900020.305806459, + "T_cal": 5900020.305806459 + }, + "17.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 35661629.75609591, + "T_cal": 35661629.75609591 + }, + "17.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 37071089.71486539, + "T_cal": 37071089.71486539 + }, + "17.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 27826654.376081422, + "T_cal": 27826654.376081422 + }, + "17.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1267267.7284929086, + "T_cal": 1267267.7284929086 + }, + "17.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 45950628.699716985, + "T_cal": 45950628.699716985 + }, + "17.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 47101411.73163301, + "T_cal": 47101411.73163301 + }, + "17.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 6591848.5687160455, + "T_cal": 6591848.5687160455 + }, + "18.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 44262482.8301141, + "T_cal": 44262482.8301141 + }, + "18.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 47358700.42335912, + "T_cal": 47358700.42335912 + }, + "18.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 36108720.36611833, + "T_cal": 36108720.36611833 + }, + "18.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1298545.2947048051, + "T_cal": 1298545.2947048051 + }, + "18.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 54023091.71841389, + "T_cal": 54023091.71841389 + }, + "18.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 53605452.05604907, + "T_cal": 53605452.05604907 + }, + "18.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 7857574.265308328, + "T_cal": 7857574.265308328 + }, + "19.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 40120809.27085395, + "T_cal": 40120809.27085395 + }, + "19.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 41012606.05755037, + "T_cal": 41012606.05755037 + }, + "19.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 34915678.38305585, + "T_cal": 34915678.38305585 + }, + "19.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1415312.0230279178, + "T_cal": 1415312.0230279178 + }, + "19.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 60813241.546283916, + "T_cal": 60813241.546283916 + }, + "19.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 59115933.672877915, + "T_cal": 59115933.672877915 + }, + "19.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 8902631.456304837, + "T_cal": 8902631.456304837 + }, + "20.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 40021423.57055149, + "T_cal": 40021423.57055149 + }, + "20.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 41869567.06025338, + "T_cal": 41869567.06025338 + }, + "20.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 38920485.71341603, + "T_cal": 38920485.71341603 + }, + "20.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1649825.3702421421, + "T_cal": 1649825.3702421421 + }, + "20.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 64361468.134043604, + "T_cal": 64361468.134043604 + }, + "20.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 61578363.26017566, + "T_cal": 61578363.26017566 + }, + "20.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 10765047.10220674, + "T_cal": 10765047.10220674 + }, + "21.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 46056327.42471032, + "T_cal": 46056327.42471032 + }, + "21.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 47514018.71081908, + "T_cal": 47514018.71081908 + }, + "21.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 45736077.4399346, + "T_cal": 45736077.4399346 + }, + "21.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1620866.9804216484, + "T_cal": 1620866.9804216484 + }, + "21.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 72686431.64429738, + "T_cal": 72686431.64429738 + }, + "21.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 67933721.33573113, + "T_cal": 67933721.33573113 + }, + "21.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 11967866.125010494, + "T_cal": 11967866.125010494 + }, + "22.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 53669040.68229101, + "T_cal": 53669040.68229101 + }, + "22.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 56100080.37376601, + "T_cal": 56100080.37376601 + }, + "22.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 51042261.26254837, + "T_cal": 51042261.26254837 + }, + "22.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 2296610.660116745, + "T_cal": 2296610.660116745 + }, + "22.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 80759725.48447388, + "T_cal": 80759725.48447388 + }, + "22.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 74305016.109796, + "T_cal": 74305016.109796 + }, + "22.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 13962605.340496987, + "T_cal": 13962605.340496987 + }, + "23.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 71733456.54175282, + "T_cal": 71733456.54175282 + }, + "23.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 72682834.70274636, + "T_cal": 72682834.70274636 + }, + "23.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 65379315.27277463, + "T_cal": 65379315.27277463 + }, + "23.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 2652651.05691829, + "T_cal": 2652651.05691829 + }, + "23.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 91688011.6645034, + "T_cal": 91688011.6645034 + }, + "23.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 84685515.19548278, + "T_cal": 84685515.19548278 + }, + "23.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 15339729.044136003, + "T_cal": 15339729.044136003 + }, + "24.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 52774900.28348556, + "T_cal": 52774900.28348556 + }, + "24.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 54500270.69390204, + "T_cal": 54500270.69390204 + }, + "24.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 62543865.57161525, + "T_cal": 62543865.57161525 + }, + "24.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 2769702.3254641984, + "T_cal": 2769702.3254641984 + }, + "24.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 101819891.95005424, + "T_cal": 101819891.95005424 + }, + "24.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 94119324.0033534, + "T_cal": 94119324.0033534 + }, + "24.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 16728253.662133003, + "T_cal": 16728253.662133003 + }, + "25.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 80304909.3567728, + "T_cal": 80304909.3567728 + }, + "25.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 83671771.69110829, + "T_cal": 83671771.69110829 + }, + "25.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 84978567.1844858, + "T_cal": 84978567.1844858 + }, + "25.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 2505064.5035228767, + "T_cal": 2505064.5035228767 + }, + "25.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 112273023.56164908, + "T_cal": 112273023.56164908 + }, + "25.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 103882977.35053928, + "T_cal": 103882977.35053928 + }, + "25.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 17746580.701786302, + "T_cal": 17746580.701786302 + }, + "26.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 64989281.26137764, + "T_cal": 64989281.26137764 + }, + "26.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 68544427.87627023, + "T_cal": 68544427.87627023 + }, + "26.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 80749609.72264951, + "T_cal": 80749609.72264951 + }, + "26.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 3133033.439591335, + "T_cal": 3133033.439591335 + }, + "26.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 123590172.37534072, + "T_cal": 123590172.37534072 + }, + "26.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 114823489.34381005, + "T_cal": 114823489.34381005 + }, + "26.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 19871728.13040249, + "T_cal": 19871728.13040249 + }, + "27.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 94368206.97103219, + "T_cal": 94368206.97103219 + }, + "27.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 99421408.9117718, + "T_cal": 99421408.9117718 + }, + "27.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 93526566.1076138, + "T_cal": 93526566.1076138 + }, + "27.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 3948527.6605453664, + "T_cal": 3948527.6605453664 + }, + "27.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 136506286.86790967, + "T_cal": 136506286.86790967 + }, + "27.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 127240802.06372823, + "T_cal": 127240802.06372823 + }, + "27.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 22736701.196728304, + "T_cal": 22736701.196728304 + }, + "28.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 85316773.38197857, + "T_cal": 85316773.38197857 + }, + "28.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 91421811.6143829, + "T_cal": 91421811.6143829 + }, + "28.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 102262225.13609418, + "T_cal": 102262225.13609418 + }, + "28.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 4013078.873440261, + "T_cal": 4013078.873440261 + }, + "28.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 141518945.1356802, + "T_cal": 141518945.1356802 + }, + "28.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 135118073.44259545, + "T_cal": 135118073.44259545 + }, + "28.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 26809475.799342208, + "T_cal": 26809475.799342208 + }, + "29.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 64772457.532240584, + "T_cal": 64772457.532240584 + }, + "29.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 68479657.3573959, + "T_cal": 68479657.3573959 + }, + "29.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 93576765.40279621, + "T_cal": 93576765.40279621 + }, + "29.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 4835487.9449910745, + "T_cal": 4835487.9449910745 + }, + "29.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 146378436.530485, + "T_cal": 146378436.530485 + }, + "29.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 140157066.28008044, + "T_cal": 140157066.28008044 + }, + "29.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 31000760.265198253, + "T_cal": 31000760.265198253 + }, + "30.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 80771514.65176713, + "T_cal": 80771514.65176713 + }, + "30.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 85746423.79299718, + "T_cal": 85746423.79299718 + }, + "30.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 107776036.21864706, + "T_cal": 107776036.21864706 + }, + "30.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 4854978.800990951, + "T_cal": 4854978.800990951 + }, + "30.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 146282215.26653653, + "T_cal": 146282215.26653653 + }, + "30.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 136137174.80119932, + "T_cal": 136137174.80119932 + }, + "30.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 35703110.116036445, + "T_cal": 35703110.116036445 + }, + "31.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 40883846.09135019, + "T_cal": 40883846.09135019 + }, + "31.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 45729561.20536516, + "T_cal": 45729561.20536516 + }, + "31.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 54000274.540750474, + "T_cal": 54000274.540750474 + }, + "31.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 4217365.56733007, + "T_cal": 4217365.56733007 + }, + "31.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 115498444.60500988, + "T_cal": 115498444.60500988 + }, + "31.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 104691453.2407249, + "T_cal": 104691453.2407249 + }, + "31.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 39205726.64051367, + "T_cal": 39205726.64051367 + } + }, + "calib_spec": "wikitext2:256", + "nsamples": 256, + "seed": 42, + "n_calib_sequences": 255 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..dc592da --- /dev/null +++ b/config.json @@ -0,0 +1,30 @@ +{ + "_name_or_path": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 11008, + "max_position_embeddings": 4096, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 32, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.46.3", + "use_cache": true, + "vocab_size": 32000 +} diff --git a/evaluation/safety/overrefusal.json b/evaluation/safety/overrefusal.json new file mode 100644 index 0000000..c0f739f --- /dev/null +++ b/evaluation/safety/overrefusal.json @@ -0,0 +1,284 @@ +{ + "tag": "l2_base_k0_a1p0_free_remove20", + "model": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove20/checkpoint", + "judge_model": "/lustre/jeesup0103/svd-safety/assets/models/Wildguard", + "judge": "WildGuard response-refusal classifier", + "protocol": "chat_template_greedy_then_wildguard", + "max_new_tokens": 512, + "limit_per_benchmark": null, + "datasets": { + "xstest_safe": { + "path": "/home/jeesup0103/svd_safety/data/xstest_safe.jsonl", + "n": 250, + "n_judge_parsed": 250, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 250, + "scored_fraction": 1.0, + "n_refused": 66, + "refusal_rate": 0.264, + "reliable": true, + "by_category": { + "definitions": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 0, + "refusal_rate": 0.0, + "reliable": true + }, + "figurative_language": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 5, + "refusal_rate": 0.2, + "reliable": true + }, + "historical_events": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 1, + "refusal_rate": 0.04, + "reliable": true + }, + "homonyms": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 7, + "refusal_rate": 0.28, + "reliable": true + }, + "nons_group_real_discr": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 14, + "refusal_rate": 0.56, + "reliable": true + }, + "privacy_fictional": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 15, + "refusal_rate": 0.6, + "reliable": true + }, + "privacy_public": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 2, + "refusal_rate": 0.08, + "reliable": true + }, + "real_group_nons_discr": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 7, + "refusal_rate": 0.28, + "reliable": true + }, + "safe_contexts": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 8, + "refusal_rate": 0.32, + "reliable": true + }, + "safe_targets": { + "n": 25, + "n_judge_parsed": 25, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 25, + "scored_fraction": 1.0, + "n_refused": 7, + "refusal_rate": 0.28, + "reliable": true + } + } + }, + "orbench_hard_1k": { + "path": "/home/jeesup0103/svd_safety/data/or_bench_hard_1k.csv", + "n": 1319, + "n_judge_parsed": 1319, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 1319, + "scored_fraction": 1.0, + "n_refused": 670, + "refusal_rate": 0.5079605761940864, + "reliable": true, + "by_category": { + "deception": { + "n": 72, + "n_judge_parsed": 72, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 72, + "scored_fraction": 1.0, + "n_refused": 37, + "refusal_rate": 0.5138888888888888, + "reliable": true + }, + "harassment": { + "n": 41, + "n_judge_parsed": 41, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 41, + "scored_fraction": 1.0, + "n_refused": 20, + "refusal_rate": 0.4878048780487805, + "reliable": true + }, + "harmful": { + "n": 106, + "n_judge_parsed": 106, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 106, + "scored_fraction": 1.0, + "n_refused": 49, + "refusal_rate": 0.46226415094339623, + "reliable": true + }, + "hate": { + "n": 54, + "n_judge_parsed": 54, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 54, + "scored_fraction": 1.0, + "n_refused": 30, + "refusal_rate": 0.5555555555555556, + "reliable": true + }, + "illegal": { + "n": 527, + "n_judge_parsed": 527, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 527, + "scored_fraction": 1.0, + "n_refused": 284, + "refusal_rate": 0.538899430740038, + "reliable": true + }, + "privacy": { + "n": 199, + "n_judge_parsed": 199, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 199, + "scored_fraction": 1.0, + "n_refused": 104, + "refusal_rate": 0.5226130653266332, + "reliable": true + }, + "self-harm": { + "n": 63, + "n_judge_parsed": 63, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 63, + "scored_fraction": 1.0, + "n_refused": 33, + "refusal_rate": 0.5238095238095238, + "reliable": true + }, + "sexual": { + "n": 66, + "n_judge_parsed": 66, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 66, + "scored_fraction": 1.0, + "n_refused": 9, + "refusal_rate": 0.13636363636363635, + "reliable": true + }, + "unethical": { + "n": 125, + "n_judge_parsed": 125, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 125, + "scored_fraction": 1.0, + "n_refused": 66, + "refusal_rate": 0.528, + "reliable": true + }, + "violence": { + "n": 66, + "n_judge_parsed": 66, + "n_judge_parse_error": 0, + "judge_parse_fraction": 1.0, + "n_uninformative": 0, + "n_scored": 66, + "scored_fraction": 1.0, + "n_refused": 38, + "refusal_rate": 0.5757575757575758, + "reliable": true + } + } + } + }, + "macro_over_refusal_rate": 0.3859802880970432, + "reliable": true, + "note": "Refusal rates use WildGuard labels over informative responses. Read reliable/scored_fraction first: compressed models that emit uninterpretable output do not receive an artificially low over-refusal score. Macro rate weights the two benchmarks equally." +} diff --git a/evaluation/safety/strongreject_summary.json b/evaluation/safety/strongreject_summary.json new file mode 100644 index 0000000..cf32105 --- /dev/null +++ b/evaluation/safety/strongreject_summary.json @@ -0,0 +1,13 @@ +{ + "tag": "l2_base_k0_a1p0_free_remove20", + "judge_model": "/lustre/jeesup0103/svd-safety/assets/models/HarmBench-Llama-2-13b-cls", + "protocol": "strongreject_direct_chat_greedy", + "n_prompts": 313, + "n_harmful": 5, + "n_not_harmful": 308, + "attack_success_rate": 0.01597444089456869, + "mean_p_yes": 0.01704984923629035, + "argmax_offlist_rate": 0.0, + "per_prompt_jsonl": "/lustre/jeesup0103/svd-safety/results/resmix/evaluation_ft/l2_base_k0_a1p0_free_remove20/safety/strongreject_per_prompt.jsonl", + "per_prompt_csv": "/lustre/jeesup0103/svd-safety/results/resmix/evaluation_ft/l2_base_k0_a1p0_free_remove20/safety/strongreject_per_prompt.csv" +} diff --git a/evaluation/safety/summary.json b/evaluation/safety/summary.json new file mode 100644 index 0000000..12ec39d --- /dev/null +++ b/evaluation/safety/summary.json @@ -0,0 +1,13 @@ +{ + "tag": "l2_base_k0_a1p0_free_remove20", + "judge_model": "/lustre/jeesup0103/svd-safety/assets/models/HarmBench-Llama-2-13b-cls", + "protocol": "advbench_direct_chat_greedy", + "n_prompts": 520, + "n_harmful": 7, + "n_not_harmful": 513, + "attack_success_rate": 0.013461538461538462, + "mean_p_yes": 0.012757582136350611, + "argmax_offlist_rate": 0.0, + "per_prompt_jsonl": "/lustre/jeesup0103/svd-safety/results/resmix/evaluation_ft/l2_base_k0_a1p0_free_remove20/safety/per_prompt.jsonl", + "per_prompt_csv": "/lustre/jeesup0103/svd-safety/results/resmix/evaluation_ft/l2_base_k0_a1p0_free_remove20/safety/per_prompt.csv" +} diff --git a/evaluation/utility/ppl.json b/evaluation/utility/ppl.json new file mode 100644 index 0000000..5058a2c --- /dev/null +++ b/evaluation/utility/ppl.json @@ -0,0 +1,6 @@ +{ + "tag": "ppl", + "model": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove20/checkpoint", + "wikitext2_ppl": 8.768001556396484, + "n_tokens": 341302 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..aaaeef2 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,10 @@ +{ + "bos_token_id": 1, + "do_sample": true, + "eos_token_id": 2, + "max_length": 4096, + "pad_token_id": 0, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.46.3" +} diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000..808c9f0 --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f232ac445053c105bdb19a6fdac7b2c9881c4282ad9bf14afb52aec02b17398 +size 4938985352 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000..4625ec6 --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deab3449fe17e37366e8ecc2a7e65513e9b67a974eed9f25dee4196153c2e9e4 +size 4947390880 diff --git a/model-00003-of-00003.safetensors b/model-00003-of-00003.safetensors new file mode 100644 index 0000000..a6e9b2c --- /dev/null +++ b/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b5935ef8d997f0e02e868f34783024fc193febad03d01984740912af495ac2f +size 3590488816 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..13674e5 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 13476831232 + }, + "weight_map": { + "lm_head.weight": "model-00003-of-00003.safetensors", + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/resmix/compression.json b/resmix/compression.json new file mode 100644 index 0000000..05588bf --- /dev/null +++ b/resmix/compression.json @@ -0,0 +1,1850 @@ +{ + "source": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", + "upstream": "AIoT-MLSys-Lab/SVD-LLM", + "step": "whiten", + "ratio_removed": 0.2, + "internal_ratio_kept": 0.8, + "nsamples": 256, + "seed": 42, + "calib_spec": "wikitext2:256", + "calib": [ + { + "name": "wikitext2", + "n": 255, + "source": "upstream get_calib_train_data" + } + ], + "n_calib_sequences": 255, + "rank_by": "resmix", + "whitening": { + "lam": 0.0, + "norm": "token", + "ridge": 0.0, + "token_mask": "nospecial", + "utility_cache": "/lustre/jeesup0103/svd-safety/work/whiten_cache/utility_wikitext256", + "utility_calib": "wikitext2:256", + "utility_tokens": 522240 + }, + "whiten_lam": 0.0, + "lam": 0.0, + "fisher_scores": null, + "fisher_num": null, + "fisher_den": null, + "fisher_form": null, + "fisher_control": null, + "fisher_floor": null, + "safety_calib": null, + "resmix_k": 0.0, + "resmix_k_abs": null, + "resmix_alpha": 1.0, + "n_safety_tokens": null, + "ranking": { + "n_matrices": 224, + "alpha": 1.0, + "k_s_total": 0, + "k_s_frac_of_rank": 0.0, + "utility_error_ratio": 1.0, + "resmix_k": 0.0, + "resmix_k_abs": null, + "resmix_alpha": 1.0 + }, + "ranking_per_matrix": { + "0.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 20.533438156264545, + "T_cal": 20.533438156264545 + }, + "0.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 18.145091391082453, + "T_cal": 18.145091391082453 + }, + "0.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1367.2978993583235, + "T_cal": 1367.2978993583235 + }, + "0.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1.7652817030071697, + "T_cal": 1.7652817030071697 + }, + "0.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 147005.47186107977, + "T_cal": 147005.47186107977 + }, + "0.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 151270.08798223134, + "T_cal": 151270.08798223134 + }, + "0.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 3251.7172484408165, + "T_cal": 3251.7172484408165 + }, + "1.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 15810.360461051278, + "T_cal": 15810.360461051278 + }, + "1.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 11679.950873318438, + "T_cal": 11679.950873318438 + }, + "1.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 34601.29388128947, + "T_cal": 34601.29388128947 + }, + "1.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 210.1460327002143, + "T_cal": 210.1460327002143 + }, + "1.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 554807.0704461301, + "T_cal": 554807.0704461301 + }, + "1.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 632577.4831049199, + "T_cal": 632577.4831049199 + }, + "1.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 41926.18052799249, + "T_cal": 41926.18052799249 + }, + "2.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1075742.3213626854, + "T_cal": 1075742.3213626854 + }, + "2.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1061928.8193543167, + "T_cal": 1061928.8193543167 + }, + "2.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1257767.9325557249, + "T_cal": 1257767.9325557249 + }, + "2.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 16470.990055876522, + "T_cal": 16470.990055876522 + }, + "2.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 2616354.6153592253, + "T_cal": 2616354.6153592253 + }, + "2.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 2619809.05808633, + "T_cal": 2619809.05808633 + }, + "2.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 122572.30074651254, + "T_cal": 122572.30074651254 + }, + "3.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 8616199.692200687, + "T_cal": 8616199.692200687 + }, + "3.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 9252912.097098961, + "T_cal": 9252912.097098961 + }, + "3.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 5181894.684841514, + "T_cal": 5181894.684841514 + }, + "3.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 28070.226419795454, + "T_cal": 28070.226419795454 + }, + "3.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 7535360.907474672, + "T_cal": 7535360.907474672 + }, + "3.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 6997858.889793897, + "T_cal": 6997858.889793897 + }, + "3.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 216506.34547028563, + "T_cal": 216506.34547028563 + }, + "4.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 8576586.798566049, + "T_cal": 8576586.798566049 + }, + "4.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 9103491.821577858, + "T_cal": 9103491.821577858 + }, + "4.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 5447171.7095543165, + "T_cal": 5447171.7095543165 + }, + "4.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 62961.396595317005, + "T_cal": 62961.396595317005 + }, + "4.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 11368285.653579928, + "T_cal": 11368285.653579928 + }, + "4.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 10192894.986038502, + "T_cal": 10192894.986038502 + }, + "4.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 399140.58106265066, + "T_cal": 399140.58106265066 + }, + "5.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 10904378.784864234, + "T_cal": 10904378.784864234 + }, + "5.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 11716277.222077748, + "T_cal": 11716277.222077748 + }, + "5.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 6791614.363699443, + "T_cal": 6791614.363699443 + }, + "5.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 83191.98936714102, + "T_cal": 83191.98936714102 + }, + "5.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 14261817.455468014, + "T_cal": 14261817.455468014 + }, + "5.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 12898852.177619234, + "T_cal": 12898852.177619234 + }, + "5.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 605437.6144528482, + "T_cal": 605437.6144528482 + }, + "6.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 15022264.490293957, + "T_cal": 15022264.490293957 + }, + "6.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 16281336.734301325, + "T_cal": 16281336.734301325 + }, + "6.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 8978679.766875453, + "T_cal": 8978679.766875453 + }, + "6.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 146576.1915614182, + "T_cal": 146576.1915614182 + }, + "6.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 18379550.492156003, + "T_cal": 18379550.492156003 + }, + "6.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 16360762.366840478, + "T_cal": 16360762.366840478 + }, + "6.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 865823.7044643352, + "T_cal": 865823.7044643352 + }, + "7.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 16961003.97794759, + "T_cal": 16961003.97794759 + }, + "7.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 17778413.16036941, + "T_cal": 17778413.16036941 + }, + "7.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 9833500.891810104, + "T_cal": 9833500.891810104 + }, + "7.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 207621.14171330316, + "T_cal": 207621.14171330316 + }, + "7.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 21548258.973763738, + "T_cal": 21548258.973763738 + }, + "7.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 19538704.244472977, + "T_cal": 19538704.244472977 + }, + "7.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 1154651.4831785636, + "T_cal": 1154651.4831785636 + }, + "8.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 18144793.681941804, + "T_cal": 18144793.681941804 + }, + "8.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 19643662.875985958, + "T_cal": 19643662.875985958 + }, + "8.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 9723547.035088653, + "T_cal": 9723547.035088653 + }, + "8.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 309933.4441164221, + "T_cal": 309933.4441164221 + }, + "8.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 22219645.429822985, + "T_cal": 22219645.429822985 + }, + "8.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 21110504.855032086, + "T_cal": 21110504.855032086 + }, + "8.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 1407579.4474217317, + "T_cal": 1407579.4474217317 + }, + "9.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 21577254.884359382, + "T_cal": 21577254.884359382 + }, + "9.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 23354272.865979496, + "T_cal": 23354272.865979496 + }, + "9.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 10870858.616251059, + "T_cal": 10870858.616251059 + }, + "9.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 460797.9531125524, + "T_cal": 460797.9531125524 + }, + "9.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 23261039.5382552, + "T_cal": 23261039.5382552 + }, + "9.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 22760975.831206925, + "T_cal": 22760975.831206925 + }, + "9.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 1698712.7164256899, + "T_cal": 1698712.7164256899 + }, + "10.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 19142257.364713803, + "T_cal": 19142257.364713803 + }, + "10.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 21642343.533273995, + "T_cal": 21642343.533273995 + }, + "10.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 10805862.494584344, + "T_cal": 10805862.494584344 + }, + "10.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 583564.7478921877, + "T_cal": 583564.7478921877 + }, + "10.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 23722052.52198033, + "T_cal": 23722052.52198033 + }, + "10.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 23921352.30383531, + "T_cal": 23921352.30383531 + }, + "10.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 1885944.1773879488, + "T_cal": 1885944.1773879488 + }, + "11.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 19812212.040702116, + "T_cal": 19812212.040702116 + }, + "11.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 21325013.33195281, + "T_cal": 21325013.33195281 + }, + "11.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 13443340.955584655, + "T_cal": 13443340.955584655 + }, + "11.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 554587.2929435676, + "T_cal": 554587.2929435676 + }, + "11.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 26289978.176920943, + "T_cal": 26289978.176920943 + }, + "11.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 27192678.451939438, + "T_cal": 27192678.451939438 + }, + "11.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 2138458.336182276, + "T_cal": 2138458.336182276 + }, + "12.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 26566691.310539044, + "T_cal": 26566691.310539044 + }, + "12.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 28198272.399621785, + "T_cal": 28198272.399621785 + }, + "12.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 14937439.7554898, + "T_cal": 14937439.7554898 + }, + "12.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 744547.0841763078, + "T_cal": 744547.0841763078 + }, + "12.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 28233288.38660199, + "T_cal": 28233288.38660199 + }, + "12.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 29627056.858521752, + "T_cal": 29627056.858521752 + }, + "12.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 2484347.4290266912, + "T_cal": 2484347.4290266912 + }, + "13.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 28594427.825724967, + "T_cal": 28594427.825724967 + }, + "13.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 32408434.376211423, + "T_cal": 32408434.376211423 + }, + "13.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 19320563.55230657, + "T_cal": 19320563.55230657 + }, + "13.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 760267.6486922672, + "T_cal": 760267.6486922672 + }, + "13.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 29710529.160073094, + "T_cal": 29710529.160073094 + }, + "13.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 32093225.825580716, + "T_cal": 32093225.825580716 + }, + "13.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 3047495.20318145, + "T_cal": 3047495.20318145 + }, + "14.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 26564102.81909692, + "T_cal": 26564102.81909692 + }, + "14.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 28371669.477075536, + "T_cal": 28371669.477075536 + }, + "14.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 18403309.45612433, + "T_cal": 18403309.45612433 + }, + "14.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 961585.4964648974, + "T_cal": 961585.4964648974 + }, + "14.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 32568338.730469096, + "T_cal": 32568338.730469096 + }, + "14.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 35260137.34940484, + "T_cal": 35260137.34940484 + }, + "14.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 3590868.026428486, + "T_cal": 3590868.026428486 + }, + "15.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 28171399.800540484, + "T_cal": 28171399.800540484 + }, + "15.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 30588812.300417174, + "T_cal": 30588812.300417174 + }, + "15.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 20111418.348540258, + "T_cal": 20111418.348540258 + }, + "15.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1140757.0870852754, + "T_cal": 1140757.0870852754 + }, + "15.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 35599225.039749816, + "T_cal": 35599225.039749816 + }, + "15.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 38562927.699108556, + "T_cal": 38562927.699108556 + }, + "15.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 4532126.281073529, + "T_cal": 4532126.281073529 + }, + "16.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 28665105.452431396, + "T_cal": 28665105.452431396 + }, + "16.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 30869692.599226408, + "T_cal": 30869692.599226408 + }, + "16.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 25487438.164353833, + "T_cal": 25487438.164353833 + }, + "16.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1505170.2068931672, + "T_cal": 1505170.2068931672 + }, + "16.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 38665250.473643295, + "T_cal": 38665250.473643295 + }, + "16.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 41007811.453693196, + "T_cal": 41007811.453693196 + }, + "16.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 5900020.305806459, + "T_cal": 5900020.305806459 + }, + "17.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 35661629.75609591, + "T_cal": 35661629.75609591 + }, + "17.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 37071089.71486539, + "T_cal": 37071089.71486539 + }, + "17.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 27826654.376081422, + "T_cal": 27826654.376081422 + }, + "17.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1267267.7284929086, + "T_cal": 1267267.7284929086 + }, + "17.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 45950628.699716985, + "T_cal": 45950628.699716985 + }, + "17.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 47101411.73163301, + "T_cal": 47101411.73163301 + }, + "17.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 6591848.5687160455, + "T_cal": 6591848.5687160455 + }, + "18.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 44262482.8301141, + "T_cal": 44262482.8301141 + }, + "18.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 47358700.42335912, + "T_cal": 47358700.42335912 + }, + "18.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 36108720.36611833, + "T_cal": 36108720.36611833 + }, + "18.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1298545.2947048051, + "T_cal": 1298545.2947048051 + }, + "18.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 54023091.71841389, + "T_cal": 54023091.71841389 + }, + "18.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 53605452.05604907, + "T_cal": 53605452.05604907 + }, + "18.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 7857574.265308328, + "T_cal": 7857574.265308328 + }, + "19.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 40120809.27085395, + "T_cal": 40120809.27085395 + }, + "19.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 41012606.05755037, + "T_cal": 41012606.05755037 + }, + "19.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 34915678.38305585, + "T_cal": 34915678.38305585 + }, + "19.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1415312.0230279178, + "T_cal": 1415312.0230279178 + }, + "19.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 60813241.546283916, + "T_cal": 60813241.546283916 + }, + "19.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 59115933.672877915, + "T_cal": 59115933.672877915 + }, + "19.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 8902631.456304837, + "T_cal": 8902631.456304837 + }, + "20.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 40021423.57055149, + "T_cal": 40021423.57055149 + }, + "20.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 41869567.06025338, + "T_cal": 41869567.06025338 + }, + "20.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 38920485.71341603, + "T_cal": 38920485.71341603 + }, + "20.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1649825.3702421421, + "T_cal": 1649825.3702421421 + }, + "20.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 64361468.134043604, + "T_cal": 64361468.134043604 + }, + "20.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 61578363.26017566, + "T_cal": 61578363.26017566 + }, + "20.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 10765047.10220674, + "T_cal": 10765047.10220674 + }, + "21.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 46056327.42471032, + "T_cal": 46056327.42471032 + }, + "21.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 47514018.71081908, + "T_cal": 47514018.71081908 + }, + "21.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 45736077.4399346, + "T_cal": 45736077.4399346 + }, + "21.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 1620866.9804216484, + "T_cal": 1620866.9804216484 + }, + "21.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 72686431.64429738, + "T_cal": 72686431.64429738 + }, + "21.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 67933721.33573113, + "T_cal": 67933721.33573113 + }, + "21.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 11967866.125010494, + "T_cal": 11967866.125010494 + }, + "22.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 53669040.68229101, + "T_cal": 53669040.68229101 + }, + "22.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 56100080.37376601, + "T_cal": 56100080.37376601 + }, + "22.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 51042261.26254837, + "T_cal": 51042261.26254837 + }, + "22.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 2296610.660116745, + "T_cal": 2296610.660116745 + }, + "22.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 80759725.48447388, + "T_cal": 80759725.48447388 + }, + "22.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 74305016.109796, + "T_cal": 74305016.109796 + }, + "22.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 13962605.340496987, + "T_cal": 13962605.340496987 + }, + "23.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 71733456.54175282, + "T_cal": 71733456.54175282 + }, + "23.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 72682834.70274636, + "T_cal": 72682834.70274636 + }, + "23.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 65379315.27277463, + "T_cal": 65379315.27277463 + }, + "23.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 2652651.05691829, + "T_cal": 2652651.05691829 + }, + "23.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 91688011.6645034, + "T_cal": 91688011.6645034 + }, + "23.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 84685515.19548278, + "T_cal": 84685515.19548278 + }, + "23.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 15339729.044136003, + "T_cal": 15339729.044136003 + }, + "24.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 52774900.28348556, + "T_cal": 52774900.28348556 + }, + "24.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 54500270.69390204, + "T_cal": 54500270.69390204 + }, + "24.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 62543865.57161525, + "T_cal": 62543865.57161525 + }, + "24.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 2769702.3254641984, + "T_cal": 2769702.3254641984 + }, + "24.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 101819891.95005424, + "T_cal": 101819891.95005424 + }, + "24.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 94119324.0033534, + "T_cal": 94119324.0033534 + }, + "24.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 16728253.662133003, + "T_cal": 16728253.662133003 + }, + "25.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 80304909.3567728, + "T_cal": 80304909.3567728 + }, + "25.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 83671771.69110829, + "T_cal": 83671771.69110829 + }, + "25.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 84978567.1844858, + "T_cal": 84978567.1844858 + }, + "25.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 2505064.5035228767, + "T_cal": 2505064.5035228767 + }, + "25.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 112273023.56164908, + "T_cal": 112273023.56164908 + }, + "25.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 103882977.35053928, + "T_cal": 103882977.35053928 + }, + "25.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 17746580.701786302, + "T_cal": 17746580.701786302 + }, + "26.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 64989281.26137764, + "T_cal": 64989281.26137764 + }, + "26.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 68544427.87627023, + "T_cal": 68544427.87627023 + }, + "26.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 80749609.72264951, + "T_cal": 80749609.72264951 + }, + "26.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 3133033.439591335, + "T_cal": 3133033.439591335 + }, + "26.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 123590172.37534072, + "T_cal": 123590172.37534072 + }, + "26.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 114823489.34381005, + "T_cal": 114823489.34381005 + }, + "26.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 19871728.13040249, + "T_cal": 19871728.13040249 + }, + "27.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 94368206.97103219, + "T_cal": 94368206.97103219 + }, + "27.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 99421408.9117718, + "T_cal": 99421408.9117718 + }, + "27.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 93526566.1076138, + "T_cal": 93526566.1076138 + }, + "27.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 3948527.6605453664, + "T_cal": 3948527.6605453664 + }, + "27.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 136506286.86790967, + "T_cal": 136506286.86790967 + }, + "27.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 127240802.06372823, + "T_cal": 127240802.06372823 + }, + "27.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 22736701.196728304, + "T_cal": 22736701.196728304 + }, + "28.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 85316773.38197857, + "T_cal": 85316773.38197857 + }, + "28.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 91421811.6143829, + "T_cal": 91421811.6143829 + }, + "28.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 102262225.13609418, + "T_cal": 102262225.13609418 + }, + "28.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 4013078.873440261, + "T_cal": 4013078.873440261 + }, + "28.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 141518945.1356802, + "T_cal": 141518945.1356802 + }, + "28.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 135118073.44259545, + "T_cal": 135118073.44259545 + }, + "28.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 26809475.799342208, + "T_cal": 26809475.799342208 + }, + "29.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 64772457.532240584, + "T_cal": 64772457.532240584 + }, + "29.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 68479657.3573959, + "T_cal": 68479657.3573959 + }, + "29.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 93576765.40279621, + "T_cal": 93576765.40279621 + }, + "29.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 4835487.9449910745, + "T_cal": 4835487.9449910745 + }, + "29.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 146378436.530485, + "T_cal": 146378436.530485 + }, + "29.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 140157066.28008044, + "T_cal": 140157066.28008044 + }, + "29.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 31000760.265198253, + "T_cal": 31000760.265198253 + }, + "30.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 80771514.65176713, + "T_cal": 80771514.65176713 + }, + "30.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 85746423.79299718, + "T_cal": 85746423.79299718 + }, + "30.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 107776036.21864706, + "T_cal": 107776036.21864706 + }, + "30.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 4854978.800990951, + "T_cal": 4854978.800990951 + }, + "30.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 146282215.26653653, + "T_cal": 146282215.26653653 + }, + "30.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 136137174.80119932, + "T_cal": 136137174.80119932 + }, + "30.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 35703110.116036445, + "T_cal": 35703110.116036445 + }, + "31.q_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 40883846.09135019, + "T_cal": 40883846.09135019 + }, + "31.k_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 45729561.20536516, + "T_cal": 45729561.20536516 + }, + "31.v_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 54000274.540750474, + "T_cal": 54000274.540750474 + }, + "31.o_proj": { + "r": 1638, + "k_u": 1638, + "k_s": 0, + "T_cal_prefix": 4217365.56733007, + "T_cal": 4217365.56733007 + }, + "31.gate_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 115498444.60500988, + "T_cal": 115498444.60500988 + }, + "31.up_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 104691453.2407249, + "T_cal": 104691453.2407249 + }, + "31.down_proj": { + "r": 2388, + "k_u": 2388, + "k_s": 0, + "T_cal_prefix": 39205726.64051367, + "T_cal": 39205726.64051367 + } + }, + "ranks": { + "0.q_proj": 1638, + "0.k_proj": 1638, + "0.v_proj": 1638, + "0.o_proj": 1638, + "0.gate_proj": 2388, + "0.up_proj": 2388, + "0.down_proj": 2388, + "1.q_proj": 1638, + "1.k_proj": 1638, + "1.v_proj": 1638, + "1.o_proj": 1638, + "1.gate_proj": 2388, + "1.up_proj": 2388, + "1.down_proj": 2388, + "2.q_proj": 1638, + "2.k_proj": 1638, + "2.v_proj": 1638, + "2.o_proj": 1638, + "2.gate_proj": 2388, + "2.up_proj": 2388, + "2.down_proj": 2388, + "3.q_proj": 1638, + "3.k_proj": 1638, + "3.v_proj": 1638, + "3.o_proj": 1638, + "3.gate_proj": 2388, + "3.up_proj": 2388, + "3.down_proj": 2388, + "4.q_proj": 1638, + "4.k_proj": 1638, + "4.v_proj": 1638, + "4.o_proj": 1638, + "4.gate_proj": 2388, + "4.up_proj": 2388, + "4.down_proj": 2388, + "5.q_proj": 1638, + "5.k_proj": 1638, + "5.v_proj": 1638, + "5.o_proj": 1638, + "5.gate_proj": 2388, + "5.up_proj": 2388, + "5.down_proj": 2388, + "6.q_proj": 1638, + "6.k_proj": 1638, + "6.v_proj": 1638, + "6.o_proj": 1638, + "6.gate_proj": 2388, + "6.up_proj": 2388, + "6.down_proj": 2388, + "7.q_proj": 1638, + "7.k_proj": 1638, + "7.v_proj": 1638, + "7.o_proj": 1638, + "7.gate_proj": 2388, + "7.up_proj": 2388, + "7.down_proj": 2388, + "8.q_proj": 1638, + "8.k_proj": 1638, + "8.v_proj": 1638, + "8.o_proj": 1638, + "8.gate_proj": 2388, + "8.up_proj": 2388, + "8.down_proj": 2388, + "9.q_proj": 1638, + "9.k_proj": 1638, + "9.v_proj": 1638, + "9.o_proj": 1638, + "9.gate_proj": 2388, + "9.up_proj": 2388, + "9.down_proj": 2388, + "10.q_proj": 1638, + "10.k_proj": 1638, + "10.v_proj": 1638, + "10.o_proj": 1638, + "10.gate_proj": 2388, + "10.up_proj": 2388, + "10.down_proj": 2388, + "11.q_proj": 1638, + "11.k_proj": 1638, + "11.v_proj": 1638, + "11.o_proj": 1638, + "11.gate_proj": 2388, + "11.up_proj": 2388, + "11.down_proj": 2388, + "12.q_proj": 1638, + "12.k_proj": 1638, + "12.v_proj": 1638, + "12.o_proj": 1638, + "12.gate_proj": 2388, + "12.up_proj": 2388, + "12.down_proj": 2388, + "13.q_proj": 1638, + "13.k_proj": 1638, + "13.v_proj": 1638, + "13.o_proj": 1638, + "13.gate_proj": 2388, + "13.up_proj": 2388, + "13.down_proj": 2388, + "14.q_proj": 1638, + "14.k_proj": 1638, + "14.v_proj": 1638, + "14.o_proj": 1638, + "14.gate_proj": 2388, + "14.up_proj": 2388, + "14.down_proj": 2388, + "15.q_proj": 1638, + "15.k_proj": 1638, + "15.v_proj": 1638, + "15.o_proj": 1638, + "15.gate_proj": 2388, + "15.up_proj": 2388, + "15.down_proj": 2388, + "16.q_proj": 1638, + "16.k_proj": 1638, + "16.v_proj": 1638, + "16.o_proj": 1638, + "16.gate_proj": 2388, + "16.up_proj": 2388, + "16.down_proj": 2388, + "17.q_proj": 1638, + "17.k_proj": 1638, + "17.v_proj": 1638, + "17.o_proj": 1638, + "17.gate_proj": 2388, + "17.up_proj": 2388, + "17.down_proj": 2388, + "18.q_proj": 1638, + "18.k_proj": 1638, + "18.v_proj": 1638, + "18.o_proj": 1638, + "18.gate_proj": 2388, + "18.up_proj": 2388, + "18.down_proj": 2388, + "19.q_proj": 1638, + "19.k_proj": 1638, + "19.v_proj": 1638, + "19.o_proj": 1638, + "19.gate_proj": 2388, + "19.up_proj": 2388, + "19.down_proj": 2388, + "20.q_proj": 1638, + "20.k_proj": 1638, + "20.v_proj": 1638, + "20.o_proj": 1638, + "20.gate_proj": 2388, + "20.up_proj": 2388, + "20.down_proj": 2388, + "21.q_proj": 1638, + "21.k_proj": 1638, + "21.v_proj": 1638, + "21.o_proj": 1638, + "21.gate_proj": 2388, + "21.up_proj": 2388, + "21.down_proj": 2388, + "22.q_proj": 1638, + "22.k_proj": 1638, + "22.v_proj": 1638, + "22.o_proj": 1638, + "22.gate_proj": 2388, + "22.up_proj": 2388, + "22.down_proj": 2388, + "23.q_proj": 1638, + "23.k_proj": 1638, + "23.v_proj": 1638, + "23.o_proj": 1638, + "23.gate_proj": 2388, + "23.up_proj": 2388, + "23.down_proj": 2388, + "24.q_proj": 1638, + "24.k_proj": 1638, + "24.v_proj": 1638, + "24.o_proj": 1638, + "24.gate_proj": 2388, + "24.up_proj": 2388, + "24.down_proj": 2388, + "25.q_proj": 1638, + "25.k_proj": 1638, + "25.v_proj": 1638, + "25.o_proj": 1638, + "25.gate_proj": 2388, + "25.up_proj": 2388, + "25.down_proj": 2388, + "26.q_proj": 1638, + "26.k_proj": 1638, + "26.v_proj": 1638, + "26.o_proj": 1638, + "26.gate_proj": 2388, + "26.up_proj": 2388, + "26.down_proj": 2388, + "27.q_proj": 1638, + "27.k_proj": 1638, + "27.v_proj": 1638, + "27.o_proj": 1638, + "27.gate_proj": 2388, + "27.up_proj": 2388, + "27.down_proj": 2388, + "28.q_proj": 1638, + "28.k_proj": 1638, + "28.v_proj": 1638, + "28.o_proj": 1638, + "28.gate_proj": 2388, + "28.up_proj": 2388, + "28.down_proj": 2388, + "29.q_proj": 1638, + "29.k_proj": 1638, + "29.v_proj": 1638, + "29.o_proj": 1638, + "29.gate_proj": 2388, + "29.up_proj": 2388, + "29.down_proj": 2388, + "30.q_proj": 1638, + "30.k_proj": 1638, + "30.v_proj": 1638, + "30.o_proj": 1638, + "30.gate_proj": 2388, + "30.up_proj": 2388, + "30.down_proj": 2388, + "31.q_proj": 1638, + "31.k_proj": 1638, + "31.v_proj": 1638, + "31.o_proj": 1638, + "31.gate_proj": 2388, + "31.up_proj": 2388, + "31.down_proj": 2388 + }, + "factored": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove20/truncated/factored.pt", + "folded": false, + "qkv_bias_restored": 0 +} \ No newline at end of file diff --git a/resmix/x_s_corpus.json b/resmix/x_s_corpus.json new file mode 100644 index 0000000..4215827 --- /dev/null +++ b/resmix/x_s_corpus.json @@ -0,0 +1,73 @@ +{ + "jbb_harm_full": { + "name": "jbb_harm_full", + "composition": "harm", + "span": "full", + "n_seqs": 11, + "seqlen": 2048, + "n_tokens": 22528, + "n_prompts_available": 100, + "n_special_per_seq": 15.545454978942871, + "source": "JailbreakBench harmful behaviours (Goal)", + "role": "harmful goal + dense model response", + "response_model": "meta-llama/Llama-2-7b-chat-hf", + "n_records": 96, + "n_kept": 96, + "min_response_chars": 20, + "dense_refusal_cue_fraction": 1.0, + "seed": 42, + "eval_overlap": { + "advbench": 11, + "strongreject": 3 + }, + "eval_overlap_subtracted": false, + "responses_from": "data/alpaca_cleaned_jbbsft_x10.json.refusals.json", + "n_units": 96, + "mean_tokens_per_unit": 264.9270833333333 + }, + "jbb_mix_full": { + "name": "jbb_mix_full", + "composition": "jbb harmful refusals + jbb benign answers", + "span": "full", + "n_seqs": 23, + "seqlen": 2048, + "n_tokens": 47104, + "n_special_per_seq": 15.0, + "harm": { + "source": "/home/jeesup0103/svd_safety/data/alpaca_cleaned_jbbsft_x10.json.refusals.json", + "n_units": 96, + "distinct_tokens": 25433, + "repeat": 1 + }, + "benign": { + "source": "/home/jeesup0103/svd_safety/data/safety_calib/jbb_benign_dense_responses.json", + "n_records": 100, + "n_refused_dropped": 54, + "n_units": 46, + "distinct_tokens": 12935, + "repeat": 2 + }, + "harm_token_share_before_packing": 0.4957, + "eval_overlap_note": "JBB harmful shares 11 AdvBench and 3 StrongREJECT goals (kept, as in the jbb arm)", + "seed": 42, + "n_units": 188, + "mean_tokens_per_unit": 272.88829787234044 + }, + "harm_full": { + "name": "harm_full", + "composition": "harm", + "span": "user turn + dense model response", + "n_seqs": 451, + "seqlen": 2048, + "n_tokens": 923648, + "n_prompts_available": 4018, + "n_special_per_seq": 16.81374740600586, + "n_units": 4018, + "mean_tokens_per_unit": 244.0955699352912, + "response_model": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", + "n_records": 4018, + "n_kept": 4018, + "min_response_chars": 20, + "dense_refusal_cue_fraction": 0.9624191139870583 + } +} \ No newline at end of file diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..451134b --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..6c00c74 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..6e95a65 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,44 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": true, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "", + "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\n' + system_message + '\\n<>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": false, + "model_max_length": 1000000000000000019884624838656, + "pad_token": null, + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +}