{ "source": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", "upstream": "AIoT-MLSys-Lab/SVD-LLM", "step": "whiten", "ratio_removed": 0.2, "internal_ratio_kept": 0.8, "nsamples": 256, "seed": 42, "calib_spec": "wikitext2:256", "calib": [ { "name": "wikitext2", "n": 255, "source": "upstream get_calib_train_data" } ], "n_calib_sequences": 255, "rank_by": "resmix", "whitening": { "lam": 0.0, "norm": "token", "ridge": 0.0, "token_mask": "nospecial", "utility_cache": "/lustre/jeesup0103/svd-safety/work/whiten_cache/utility_wikitext256", "utility_calib": "wikitext2:256", "utility_tokens": 522240 }, "whiten_lam": 0.0, "lam": 0.0, "fisher_scores": null, "fisher_num": null, "fisher_den": null, "fisher_form": null, "fisher_control": null, "fisher_floor": null, "safety_calib": null, "resmix_k": 0.0, "resmix_k_abs": null, "resmix_alpha": 1.0, "n_safety_tokens": null, "ranking": { "n_matrices": 224, "alpha": 1.0, "k_s_total": 0, "k_s_frac_of_rank": 0.0, "utility_error_ratio": 1.0, "resmix_k": 0.0, "resmix_k_abs": null, "resmix_alpha": 1.0 }, "ranking_per_matrix": { "0.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 20.533438156264545, "T_cal": 20.533438156264545 }, "0.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 18.145091391082453, "T_cal": 18.145091391082453 }, "0.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 1367.2978993583235, "T_cal": 1367.2978993583235 }, "0.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 1.7652817030071697, "T_cal": 1.7652817030071697 }, "0.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 147005.47186107977, "T_cal": 147005.47186107977 }, "0.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 151270.08798223134, "T_cal": 151270.08798223134 }, "0.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 3251.7172484408165, "T_cal": 3251.7172484408165 }, "1.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 15810.360461051278, "T_cal": 15810.360461051278 }, "1.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 11679.950873318438, "T_cal": 11679.950873318438 }, "1.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 34601.29388128947, "T_cal": 34601.29388128947 }, "1.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 210.1460327002143, "T_cal": 210.1460327002143 }, "1.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 554807.0704461301, "T_cal": 554807.0704461301 }, "1.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 632577.4831049199, "T_cal": 632577.4831049199 }, "1.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 41926.18052799249, "T_cal": 41926.18052799249 }, "2.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 1075742.3213626854, "T_cal": 1075742.3213626854 }, "2.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 1061928.8193543167, "T_cal": 1061928.8193543167 }, "2.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 1257767.9325557249, "T_cal": 1257767.9325557249 }, "2.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 16470.990055876522, "T_cal": 16470.990055876522 }, "2.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 2616354.6153592253, "T_cal": 2616354.6153592253 }, "2.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 2619809.05808633, "T_cal": 2619809.05808633 }, "2.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 122572.30074651254, "T_cal": 122572.30074651254 }, "3.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 8616199.692200687, "T_cal": 8616199.692200687 }, "3.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 9252912.097098961, "T_cal": 9252912.097098961 }, "3.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 5181894.684841514, "T_cal": 5181894.684841514 }, "3.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 28070.226419795454, "T_cal": 28070.226419795454 }, "3.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 7535360.907474672, "T_cal": 7535360.907474672 }, "3.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 6997858.889793897, "T_cal": 6997858.889793897 }, "3.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 216506.34547028563, "T_cal": 216506.34547028563 }, "4.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 8576586.798566049, "T_cal": 8576586.798566049 }, "4.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 9103491.821577858, "T_cal": 9103491.821577858 }, "4.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 5447171.7095543165, "T_cal": 5447171.7095543165 }, "4.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 62961.396595317005, "T_cal": 62961.396595317005 }, "4.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 11368285.653579928, "T_cal": 11368285.653579928 }, "4.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 10192894.986038502, "T_cal": 10192894.986038502 }, "4.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 399140.58106265066, "T_cal": 399140.58106265066 }, "5.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 10904378.784864234, "T_cal": 10904378.784864234 }, "5.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 11716277.222077748, "T_cal": 11716277.222077748 }, "5.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 6791614.363699443, "T_cal": 6791614.363699443 }, "5.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 83191.98936714102, "T_cal": 83191.98936714102 }, "5.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 14261817.455468014, "T_cal": 14261817.455468014 }, "5.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 12898852.177619234, "T_cal": 12898852.177619234 }, "5.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 605437.6144528482, "T_cal": 605437.6144528482 }, "6.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 15022264.490293957, "T_cal": 15022264.490293957 }, "6.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 16281336.734301325, "T_cal": 16281336.734301325 }, "6.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 8978679.766875453, "T_cal": 8978679.766875453 }, "6.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 146576.1915614182, "T_cal": 146576.1915614182 }, "6.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 18379550.492156003, "T_cal": 18379550.492156003 }, "6.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 16360762.366840478, "T_cal": 16360762.366840478 }, "6.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 865823.7044643352, "T_cal": 865823.7044643352 }, "7.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 16961003.97794759, "T_cal": 16961003.97794759 }, "7.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 17778413.16036941, "T_cal": 17778413.16036941 }, "7.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 9833500.891810104, "T_cal": 9833500.891810104 }, "7.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 207621.14171330316, "T_cal": 207621.14171330316 }, "7.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 21548258.973763738, "T_cal": 21548258.973763738 }, "7.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 19538704.244472977, "T_cal": 19538704.244472977 }, "7.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 1154651.4831785636, "T_cal": 1154651.4831785636 }, "8.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 18144793.681941804, "T_cal": 18144793.681941804 }, "8.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 19643662.875985958, "T_cal": 19643662.875985958 }, "8.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 9723547.035088653, "T_cal": 9723547.035088653 }, "8.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 309933.4441164221, "T_cal": 309933.4441164221 }, "8.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 22219645.429822985, "T_cal": 22219645.429822985 }, "8.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 21110504.855032086, "T_cal": 21110504.855032086 }, "8.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 1407579.4474217317, "T_cal": 1407579.4474217317 }, "9.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 21577254.884359382, "T_cal": 21577254.884359382 }, "9.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 23354272.865979496, "T_cal": 23354272.865979496 }, "9.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 10870858.616251059, "T_cal": 10870858.616251059 }, "9.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 460797.9531125524, "T_cal": 460797.9531125524 }, "9.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 23261039.5382552, "T_cal": 23261039.5382552 }, "9.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 22760975.831206925, "T_cal": 22760975.831206925 }, "9.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 1698712.7164256899, "T_cal": 1698712.7164256899 }, "10.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 19142257.364713803, "T_cal": 19142257.364713803 }, "10.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 21642343.533273995, "T_cal": 21642343.533273995 }, "10.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 10805862.494584344, "T_cal": 10805862.494584344 }, "10.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 583564.7478921877, "T_cal": 583564.7478921877 }, "10.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 23722052.52198033, "T_cal": 23722052.52198033 }, "10.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 23921352.30383531, "T_cal": 23921352.30383531 }, "10.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 1885944.1773879488, "T_cal": 1885944.1773879488 }, "11.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 19812212.040702116, "T_cal": 19812212.040702116 }, "11.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 21325013.33195281, "T_cal": 21325013.33195281 }, "11.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 13443340.955584655, "T_cal": 13443340.955584655 }, "11.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 554587.2929435676, "T_cal": 554587.2929435676 }, "11.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 26289978.176920943, "T_cal": 26289978.176920943 }, "11.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 27192678.451939438, "T_cal": 27192678.451939438 }, "11.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 2138458.336182276, "T_cal": 2138458.336182276 }, "12.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 26566691.310539044, "T_cal": 26566691.310539044 }, "12.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 28198272.399621785, "T_cal": 28198272.399621785 }, "12.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 14937439.7554898, "T_cal": 14937439.7554898 }, "12.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 744547.0841763078, "T_cal": 744547.0841763078 }, "12.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 28233288.38660199, "T_cal": 28233288.38660199 }, "12.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 29627056.858521752, "T_cal": 29627056.858521752 }, "12.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 2484347.4290266912, "T_cal": 2484347.4290266912 }, "13.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 28594427.825724967, "T_cal": 28594427.825724967 }, "13.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 32408434.376211423, "T_cal": 32408434.376211423 }, "13.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 19320563.55230657, "T_cal": 19320563.55230657 }, "13.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 760267.6486922672, "T_cal": 760267.6486922672 }, "13.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 29710529.160073094, "T_cal": 29710529.160073094 }, "13.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 32093225.825580716, "T_cal": 32093225.825580716 }, "13.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 3047495.20318145, "T_cal": 3047495.20318145 }, "14.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 26564102.81909692, "T_cal": 26564102.81909692 }, "14.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 28371669.477075536, "T_cal": 28371669.477075536 }, "14.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 18403309.45612433, "T_cal": 18403309.45612433 }, "14.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 961585.4964648974, "T_cal": 961585.4964648974 }, "14.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 32568338.730469096, "T_cal": 32568338.730469096 }, "14.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 35260137.34940484, "T_cal": 35260137.34940484 }, "14.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 3590868.026428486, "T_cal": 3590868.026428486 }, "15.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 28171399.800540484, "T_cal": 28171399.800540484 }, "15.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 30588812.300417174, "T_cal": 30588812.300417174 }, "15.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 20111418.348540258, "T_cal": 20111418.348540258 }, "15.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 1140757.0870852754, "T_cal": 1140757.0870852754 }, "15.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 35599225.039749816, "T_cal": 35599225.039749816 }, "15.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 38562927.699108556, "T_cal": 38562927.699108556 }, "15.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 4532126.281073529, "T_cal": 4532126.281073529 }, "16.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 28665105.452431396, "T_cal": 28665105.452431396 }, "16.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 30869692.599226408, "T_cal": 30869692.599226408 }, "16.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 25487438.164353833, "T_cal": 25487438.164353833 }, "16.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 1505170.2068931672, "T_cal": 1505170.2068931672 }, "16.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 38665250.473643295, "T_cal": 38665250.473643295 }, "16.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 41007811.453693196, "T_cal": 41007811.453693196 }, "16.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 5900020.305806459, "T_cal": 5900020.305806459 }, "17.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 35661629.75609591, "T_cal": 35661629.75609591 }, "17.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 37071089.71486539, "T_cal": 37071089.71486539 }, "17.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 27826654.376081422, "T_cal": 27826654.376081422 }, "17.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 1267267.7284929086, "T_cal": 1267267.7284929086 }, "17.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 45950628.699716985, "T_cal": 45950628.699716985 }, "17.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 47101411.73163301, "T_cal": 47101411.73163301 }, "17.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 6591848.5687160455, "T_cal": 6591848.5687160455 }, "18.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 44262482.8301141, "T_cal": 44262482.8301141 }, "18.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 47358700.42335912, "T_cal": 47358700.42335912 }, "18.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 36108720.36611833, "T_cal": 36108720.36611833 }, "18.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 1298545.2947048051, "T_cal": 1298545.2947048051 }, "18.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 54023091.71841389, "T_cal": 54023091.71841389 }, "18.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 53605452.05604907, "T_cal": 53605452.05604907 }, "18.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 7857574.265308328, "T_cal": 7857574.265308328 }, "19.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 40120809.27085395, "T_cal": 40120809.27085395 }, "19.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 41012606.05755037, "T_cal": 41012606.05755037 }, "19.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 34915678.38305585, "T_cal": 34915678.38305585 }, "19.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 1415312.0230279178, "T_cal": 1415312.0230279178 }, "19.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 60813241.546283916, "T_cal": 60813241.546283916 }, "19.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 59115933.672877915, "T_cal": 59115933.672877915 }, "19.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 8902631.456304837, "T_cal": 8902631.456304837 }, "20.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 40021423.57055149, "T_cal": 40021423.57055149 }, "20.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 41869567.06025338, "T_cal": 41869567.06025338 }, "20.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 38920485.71341603, "T_cal": 38920485.71341603 }, "20.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 1649825.3702421421, "T_cal": 1649825.3702421421 }, "20.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 64361468.134043604, "T_cal": 64361468.134043604 }, "20.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 61578363.26017566, "T_cal": 61578363.26017566 }, "20.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 10765047.10220674, "T_cal": 10765047.10220674 }, "21.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 46056327.42471032, "T_cal": 46056327.42471032 }, "21.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 47514018.71081908, "T_cal": 47514018.71081908 }, "21.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 45736077.4399346, "T_cal": 45736077.4399346 }, "21.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 1620866.9804216484, "T_cal": 1620866.9804216484 }, "21.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 72686431.64429738, "T_cal": 72686431.64429738 }, "21.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 67933721.33573113, "T_cal": 67933721.33573113 }, "21.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 11967866.125010494, "T_cal": 11967866.125010494 }, "22.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 53669040.68229101, "T_cal": 53669040.68229101 }, "22.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 56100080.37376601, "T_cal": 56100080.37376601 }, "22.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 51042261.26254837, "T_cal": 51042261.26254837 }, "22.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 2296610.660116745, "T_cal": 2296610.660116745 }, "22.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 80759725.48447388, "T_cal": 80759725.48447388 }, "22.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 74305016.109796, "T_cal": 74305016.109796 }, "22.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 13962605.340496987, "T_cal": 13962605.340496987 }, "23.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 71733456.54175282, "T_cal": 71733456.54175282 }, "23.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 72682834.70274636, "T_cal": 72682834.70274636 }, "23.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 65379315.27277463, "T_cal": 65379315.27277463 }, "23.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 2652651.05691829, "T_cal": 2652651.05691829 }, "23.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 91688011.6645034, "T_cal": 91688011.6645034 }, "23.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 84685515.19548278, "T_cal": 84685515.19548278 }, "23.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 15339729.044136003, "T_cal": 15339729.044136003 }, "24.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 52774900.28348556, "T_cal": 52774900.28348556 }, "24.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 54500270.69390204, "T_cal": 54500270.69390204 }, "24.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 62543865.57161525, "T_cal": 62543865.57161525 }, "24.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 2769702.3254641984, "T_cal": 2769702.3254641984 }, "24.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 101819891.95005424, "T_cal": 101819891.95005424 }, "24.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 94119324.0033534, "T_cal": 94119324.0033534 }, "24.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 16728253.662133003, "T_cal": 16728253.662133003 }, "25.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 80304909.3567728, "T_cal": 80304909.3567728 }, "25.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 83671771.69110829, "T_cal": 83671771.69110829 }, "25.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 84978567.1844858, "T_cal": 84978567.1844858 }, "25.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 2505064.5035228767, "T_cal": 2505064.5035228767 }, "25.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 112273023.56164908, "T_cal": 112273023.56164908 }, "25.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 103882977.35053928, "T_cal": 103882977.35053928 }, "25.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 17746580.701786302, "T_cal": 17746580.701786302 }, "26.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 64989281.26137764, "T_cal": 64989281.26137764 }, "26.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 68544427.87627023, "T_cal": 68544427.87627023 }, "26.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 80749609.72264951, "T_cal": 80749609.72264951 }, "26.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 3133033.439591335, "T_cal": 3133033.439591335 }, "26.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 123590172.37534072, "T_cal": 123590172.37534072 }, "26.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 114823489.34381005, "T_cal": 114823489.34381005 }, "26.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 19871728.13040249, "T_cal": 19871728.13040249 }, "27.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 94368206.97103219, "T_cal": 94368206.97103219 }, "27.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 99421408.9117718, "T_cal": 99421408.9117718 }, "27.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 93526566.1076138, "T_cal": 93526566.1076138 }, "27.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 3948527.6605453664, "T_cal": 3948527.6605453664 }, "27.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 136506286.86790967, "T_cal": 136506286.86790967 }, "27.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 127240802.06372823, "T_cal": 127240802.06372823 }, "27.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 22736701.196728304, "T_cal": 22736701.196728304 }, "28.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 85316773.38197857, "T_cal": 85316773.38197857 }, "28.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 91421811.6143829, "T_cal": 91421811.6143829 }, "28.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 102262225.13609418, "T_cal": 102262225.13609418 }, "28.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 4013078.873440261, "T_cal": 4013078.873440261 }, "28.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 141518945.1356802, "T_cal": 141518945.1356802 }, "28.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 135118073.44259545, "T_cal": 135118073.44259545 }, "28.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 26809475.799342208, "T_cal": 26809475.799342208 }, "29.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 64772457.532240584, "T_cal": 64772457.532240584 }, "29.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 68479657.3573959, "T_cal": 68479657.3573959 }, "29.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 93576765.40279621, "T_cal": 93576765.40279621 }, "29.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 4835487.9449910745, "T_cal": 4835487.9449910745 }, "29.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 146378436.530485, "T_cal": 146378436.530485 }, "29.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 140157066.28008044, "T_cal": 140157066.28008044 }, "29.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 31000760.265198253, "T_cal": 31000760.265198253 }, "30.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 80771514.65176713, "T_cal": 80771514.65176713 }, "30.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 85746423.79299718, "T_cal": 85746423.79299718 }, "30.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 107776036.21864706, "T_cal": 107776036.21864706 }, "30.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 4854978.800990951, "T_cal": 4854978.800990951 }, "30.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 146282215.26653653, "T_cal": 146282215.26653653 }, "30.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 136137174.80119932, "T_cal": 136137174.80119932 }, "30.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 35703110.116036445, "T_cal": 35703110.116036445 }, "31.q_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 40883846.09135019, "T_cal": 40883846.09135019 }, "31.k_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 45729561.20536516, "T_cal": 45729561.20536516 }, "31.v_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 54000274.540750474, "T_cal": 54000274.540750474 }, "31.o_proj": { "r": 1638, "k_u": 1638, "k_s": 0, "T_cal_prefix": 4217365.56733007, "T_cal": 4217365.56733007 }, "31.gate_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 115498444.60500988, "T_cal": 115498444.60500988 }, "31.up_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 104691453.2407249, "T_cal": 104691453.2407249 }, "31.down_proj": { "r": 2388, "k_u": 2388, "k_s": 0, "T_cal_prefix": 39205726.64051367, "T_cal": 39205726.64051367 } }, "ranks": { "0.q_proj": 1638, "0.k_proj": 1638, "0.v_proj": 1638, "0.o_proj": 1638, "0.gate_proj": 2388, "0.up_proj": 2388, "0.down_proj": 2388, "1.q_proj": 1638, "1.k_proj": 1638, "1.v_proj": 1638, "1.o_proj": 1638, "1.gate_proj": 2388, "1.up_proj": 2388, "1.down_proj": 2388, "2.q_proj": 1638, "2.k_proj": 1638, "2.v_proj": 1638, "2.o_proj": 1638, "2.gate_proj": 2388, "2.up_proj": 2388, "2.down_proj": 2388, "3.q_proj": 1638, "3.k_proj": 1638, "3.v_proj": 1638, "3.o_proj": 1638, "3.gate_proj": 2388, "3.up_proj": 2388, "3.down_proj": 2388, "4.q_proj": 1638, "4.k_proj": 1638, "4.v_proj": 1638, "4.o_proj": 1638, "4.gate_proj": 2388, "4.up_proj": 2388, "4.down_proj": 2388, "5.q_proj": 1638, "5.k_proj": 1638, "5.v_proj": 1638, "5.o_proj": 1638, "5.gate_proj": 2388, "5.up_proj": 2388, "5.down_proj": 2388, "6.q_proj": 1638, "6.k_proj": 1638, "6.v_proj": 1638, "6.o_proj": 1638, "6.gate_proj": 2388, "6.up_proj": 2388, "6.down_proj": 2388, "7.q_proj": 1638, "7.k_proj": 1638, "7.v_proj": 1638, "7.o_proj": 1638, "7.gate_proj": 2388, "7.up_proj": 2388, "7.down_proj": 2388, "8.q_proj": 1638, "8.k_proj": 1638, "8.v_proj": 1638, "8.o_proj": 1638, "8.gate_proj": 2388, "8.up_proj": 2388, "8.down_proj": 2388, "9.q_proj": 1638, "9.k_proj": 1638, "9.v_proj": 1638, "9.o_proj": 1638, "9.gate_proj": 2388, "9.up_proj": 2388, "9.down_proj": 2388, "10.q_proj": 1638, "10.k_proj": 1638, "10.v_proj": 1638, "10.o_proj": 1638, "10.gate_proj": 2388, "10.up_proj": 2388, "10.down_proj": 2388, "11.q_proj": 1638, "11.k_proj": 1638, "11.v_proj": 1638, "11.o_proj": 1638, "11.gate_proj": 2388, "11.up_proj": 2388, "11.down_proj": 2388, "12.q_proj": 1638, "12.k_proj": 1638, "12.v_proj": 1638, "12.o_proj": 1638, "12.gate_proj": 2388, "12.up_proj": 2388, "12.down_proj": 2388, "13.q_proj": 1638, "13.k_proj": 1638, "13.v_proj": 1638, "13.o_proj": 1638, "13.gate_proj": 2388, "13.up_proj": 2388, "13.down_proj": 2388, "14.q_proj": 1638, "14.k_proj": 1638, "14.v_proj": 1638, "14.o_proj": 1638, "14.gate_proj": 2388, "14.up_proj": 2388, "14.down_proj": 2388, "15.q_proj": 1638, "15.k_proj": 1638, "15.v_proj": 1638, "15.o_proj": 1638, "15.gate_proj": 2388, "15.up_proj": 2388, "15.down_proj": 2388, "16.q_proj": 1638, "16.k_proj": 1638, "16.v_proj": 1638, "16.o_proj": 1638, "16.gate_proj": 2388, "16.up_proj": 2388, "16.down_proj": 2388, "17.q_proj": 1638, "17.k_proj": 1638, "17.v_proj": 1638, "17.o_proj": 1638, "17.gate_proj": 2388, "17.up_proj": 2388, "17.down_proj": 2388, "18.q_proj": 1638, "18.k_proj": 1638, "18.v_proj": 1638, "18.o_proj": 1638, "18.gate_proj": 2388, "18.up_proj": 2388, "18.down_proj": 2388, "19.q_proj": 1638, "19.k_proj": 1638, "19.v_proj": 1638, "19.o_proj": 1638, "19.gate_proj": 2388, "19.up_proj": 2388, "19.down_proj": 2388, "20.q_proj": 1638, "20.k_proj": 1638, "20.v_proj": 1638, "20.o_proj": 1638, "20.gate_proj": 2388, "20.up_proj": 2388, "20.down_proj": 2388, "21.q_proj": 1638, "21.k_proj": 1638, "21.v_proj": 1638, "21.o_proj": 1638, "21.gate_proj": 2388, "21.up_proj": 2388, "21.down_proj": 2388, "22.q_proj": 1638, "22.k_proj": 1638, "22.v_proj": 1638, "22.o_proj": 1638, "22.gate_proj": 2388, "22.up_proj": 2388, "22.down_proj": 2388, "23.q_proj": 1638, "23.k_proj": 1638, "23.v_proj": 1638, "23.o_proj": 1638, "23.gate_proj": 2388, "23.up_proj": 2388, "23.down_proj": 2388, "24.q_proj": 1638, "24.k_proj": 1638, "24.v_proj": 1638, "24.o_proj": 1638, "24.gate_proj": 2388, "24.up_proj": 2388, "24.down_proj": 2388, "25.q_proj": 1638, "25.k_proj": 1638, "25.v_proj": 1638, "25.o_proj": 1638, "25.gate_proj": 2388, "25.up_proj": 2388, "25.down_proj": 2388, "26.q_proj": 1638, "26.k_proj": 1638, "26.v_proj": 1638, "26.o_proj": 1638, "26.gate_proj": 2388, "26.up_proj": 2388, "26.down_proj": 2388, "27.q_proj": 1638, "27.k_proj": 1638, "27.v_proj": 1638, "27.o_proj": 1638, "27.gate_proj": 2388, "27.up_proj": 2388, "27.down_proj": 2388, "28.q_proj": 1638, "28.k_proj": 1638, "28.v_proj": 1638, "28.o_proj": 1638, "28.gate_proj": 2388, "28.up_proj": 2388, "28.down_proj": 2388, "29.q_proj": 1638, "29.k_proj": 1638, "29.v_proj": 1638, "29.o_proj": 1638, "29.gate_proj": 2388, "29.up_proj": 2388, "29.down_proj": 2388, "30.q_proj": 1638, "30.k_proj": 1638, "30.v_proj": 1638, "30.o_proj": 1638, "30.gate_proj": 2388, "30.up_proj": 2388, "30.down_proj": 2388, "31.q_proj": 1638, "31.k_proj": 1638, "31.v_proj": 1638, "31.o_proj": 1638, "31.gate_proj": 2388, "31.up_proj": 2388, "31.down_proj": 2388 }, "factored": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove20/truncated/factored.pt", "folded": false, "qkv_bias_restored": 0 }