{ "source": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", "upstream": "AIoT-MLSys-Lab/SVD-LLM", "step": "whiten", "ratio_removed": 0.5, "internal_ratio_kept": 0.5, "nsamples": 256, "seed": 42, "calib_spec": "wikitext2:256", "calib": [ { "name": "wikitext2", "n": 255, "source": "upstream get_calib_train_data" } ], "n_calib_sequences": 255, "rank_by": "resmix", "whitening": { "lam": 0.0, "norm": "token", "ridge": 0.0, "token_mask": "nospecial", "utility_cache": "/lustre/jeesup0103/svd-safety/work/whiten_cache/utility_wikitext256", "utility_calib": "wikitext2:256", "utility_tokens": 522240 }, "whiten_lam": 0.0, "lam": 0.0, "fisher_scores": null, "fisher_num": null, "fisher_den": null, "fisher_form": null, "fisher_control": null, "fisher_floor": null, "safety_calib": null, "resmix_k": 0.0, "resmix_k_abs": null, "resmix_alpha": 1.0, "n_safety_tokens": null, "ranking": { "n_matrices": 224, "alpha": 1.0, "k_s_total": 0, "k_s_frac_of_rank": 0.0, "utility_error_ratio": 1.0, "resmix_k": 0.0, "resmix_k_abs": null, "resmix_alpha": 1.0 }, "ranking_per_matrix": { "0.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 156.2822172863302, "T_cal": 156.2822172863302 }, "0.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 136.6076303460885, "T_cal": 136.6076303460885 }, "0.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 5563.601948451262, "T_cal": 5563.601948451262 }, "0.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 11.883489327525915, "T_cal": 11.883489327525915 }, "0.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 551494.2880854442, "T_cal": 551494.2880854442 }, "0.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 567257.6364177521, "T_cal": 567257.6364177521 }, "0.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 10574.73416621288, "T_cal": 10574.73416621288 }, "1.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 100240.23788563588, "T_cal": 100240.23788563588 }, "1.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 80244.36699064514, "T_cal": 80244.36699064514 }, "1.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 117968.30722943979, "T_cal": 117968.30722943979 }, "1.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 2659.704093725096, "T_cal": 2659.704093725096 }, "1.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 2235436.099383505, "T_cal": 2235436.099383505 }, "1.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 2509229.3785931463, "T_cal": 2509229.3785931463 }, "1.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 109809.44940937065, "T_cal": 109809.44940937065 }, "2.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 4746539.381895847, "T_cal": 4746539.381895847 }, "2.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 4918824.6343466835, "T_cal": 4918824.6343466835 }, "2.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 3523890.5161447423, "T_cal": 3523890.5161447423 }, "2.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 50969.97357040483, "T_cal": 50969.97357040483 }, "2.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 8100129.686990686, "T_cal": 8100129.686990686 }, "2.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 8115273.942346663, "T_cal": 8115273.942346663 }, "2.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 294664.8670134115, "T_cal": 294664.8670134115 }, "3.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 26549892.624591216, "T_cal": 26549892.624591216 }, "3.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 29411829.46453424, "T_cal": 29411829.46453424 }, "3.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 12849160.788258456, "T_cal": 12849160.788258456 }, "3.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 75344.21232350322, "T_cal": 75344.21232350322 }, "3.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 20124599.87473761, "T_cal": 20124599.87473761 }, "3.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 18640112.770588867, "T_cal": 18640112.770588867 }, "3.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 524820.9066087082, "T_cal": 524820.9066087082 }, "4.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 25250542.407848857, "T_cal": 25250542.407848857 }, "4.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 27394800.24831686, "T_cal": 27394800.24831686 }, "4.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 13191767.594822232, "T_cal": 13191767.594822232 }, "4.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 169062.55273735774, "T_cal": 169062.55273735774 }, "4.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 29237100.052914195, "T_cal": 29237100.052914195 }, "4.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 26090482.406680807, "T_cal": 26090482.406680807 }, "4.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 963855.1964731526, "T_cal": 963855.1964731526 }, "5.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 30895764.9122892, "T_cal": 30895764.9122892 }, "5.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 33769890.54973752, "T_cal": 33769890.54973752 }, "5.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 16361892.980628585, "T_cal": 16361892.980628585 }, "5.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 224083.87262812065, "T_cal": 224083.87262812065 }, "5.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 36314528.86074707, "T_cal": 36314528.86074707 }, "5.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 32617453.673472613, "T_cal": 32617453.673472613 }, "5.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 1469035.2155729723, "T_cal": 1469035.2155729723 }, "6.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 41388364.19854486, "T_cal": 41388364.19854486 }, "6.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 45172994.90271932, "T_cal": 45172994.90271932 }, "6.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 21678924.804320216, "T_cal": 21678924.804320216 }, "6.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 383034.14047510386, "T_cal": 383034.14047510386 }, "6.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 46209650.42743263, "T_cal": 46209650.42743263 }, "6.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 40650261.85568227, "T_cal": 40650261.85568227 }, "6.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 2118973.784861637, "T_cal": 2118973.784861637 }, "7.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 47284137.39105231, "T_cal": 47284137.39105231 }, "7.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 49516277.76804145, "T_cal": 49516277.76804145 }, "7.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 24025444.338734645, "T_cal": 24025444.338734645 }, "7.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 529522.3713790925, "T_cal": 529522.3713790925 }, "7.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 53621336.83408186, "T_cal": 53621336.83408186 }, "7.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 47973642.76373335, "T_cal": 47973642.76373335 }, "7.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 2848496.63419468, "T_cal": 2848496.63419468 }, "8.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 50163193.01631209, "T_cal": 50163193.01631209 }, "8.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 54599051.82897397, "T_cal": 54599051.82897397 }, "8.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 23749238.527001403, "T_cal": 23749238.527001403 }, "8.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 804572.5452023039, "T_cal": 804572.5452023039 }, "8.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 55039452.52156936, "T_cal": 55039452.52156936 }, "8.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 51746108.93848659, "T_cal": 51746108.93848659 }, "8.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 3455046.1451842436, "T_cal": 3455046.1451842436 }, "9.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 58792016.99550491, "T_cal": 58792016.99550491 }, "9.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 63800549.894802615, "T_cal": 63800549.894802615 }, "9.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 26766599.90078198, "T_cal": 26766599.90078198 }, "9.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 1197485.5025786553, "T_cal": 1197485.5025786553 }, "9.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 57738994.27740162, "T_cal": 57738994.27740162 }, "9.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 55864681.50744352, "T_cal": 55864681.50744352 }, "9.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 4155296.6601606603, "T_cal": 4155296.6601606603 }, "10.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 53220474.804035515, "T_cal": 53220474.804035515 }, "10.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 60420192.90099472, "T_cal": 60420192.90099472 }, "10.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 26566691.704149634, "T_cal": 26566691.704149634 }, "10.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 1558371.52803643, "T_cal": 1558371.52803643 }, "10.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 59137207.21414365, "T_cal": 59137207.21414365 }, "10.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 58852844.26278656, "T_cal": 58852844.26278656 }, "10.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 4624176.5717368005, "T_cal": 4624176.5717368005 }, "11.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 54654227.84304168, "T_cal": 54654227.84304168 }, "11.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 58344595.5135048, "T_cal": 58344595.5135048 }, "11.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 33450970.13100764, "T_cal": 33450970.13100764 }, "11.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 1463054.384851118, "T_cal": 1463054.384851118 }, "11.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 64987101.35712747, "T_cal": 64987101.35712747 }, "11.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 66265263.73580673, "T_cal": 66265263.73580673 }, "11.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 5196261.61861577, "T_cal": 5196261.61861577 }, "12.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 71541824.16024876, "T_cal": 71541824.16024876 }, "12.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 75403037.7214964, "T_cal": 75403037.7214964 }, "12.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 36981022.303653136, "T_cal": 36981022.303653136 }, "12.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 1887717.0479989685, "T_cal": 1887717.0479989685 }, "12.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 69695343.20320588, "T_cal": 69695343.20320588 }, "12.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 72276642.03605041, "T_cal": 72276642.03605041 }, "12.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 6033882.061814375, "T_cal": 6033882.061814375 }, "13.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 75431071.33490866, "T_cal": 75431071.33490866 }, "13.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 84792981.87782457, "T_cal": 84792981.87782457 }, "13.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 46179911.510070644, "T_cal": 46179911.510070644 }, "13.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 1955216.6759422035, "T_cal": 1955216.6759422035 }, "13.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 73233695.44846965, "T_cal": 73233695.44846965 }, "13.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 78073226.56023219, "T_cal": 78073226.56023219 }, "13.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 7324186.484293869, "T_cal": 7324186.484293869 }, "14.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 70465715.42144284, "T_cal": 70465715.42144284 }, "14.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 74578770.20836243, "T_cal": 74578770.20836243 }, "14.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 44770865.05114746, "T_cal": 44770865.05114746 }, "14.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 2423284.544792806, "T_cal": 2423284.544792806 }, "14.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 80497814.28728521, "T_cal": 80497814.28728521 }, "14.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 86133112.99762905, "T_cal": 86133112.99762905 }, "14.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 8639030.741389275, "T_cal": 8639030.741389275 }, "15.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 74176470.24746722, "T_cal": 74176470.24746722 }, "15.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 79776294.2946751, "T_cal": 79776294.2946751 }, "15.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 48261780.823118016, "T_cal": 48261780.823118016 }, "15.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 2864259.540227811, "T_cal": 2864259.540227811 }, "15.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 88024559.4091621, "T_cal": 88024559.4091621 }, "15.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 94005414.8384403, "T_cal": 94005414.8384403 }, "15.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 10768412.023026569, "T_cal": 10768412.023026569 }, "16.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 74656454.28043205, "T_cal": 74656454.28043205 }, "16.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 80006787.4789646, "T_cal": 80006787.4789646 }, "16.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 59002924.25436937, "T_cal": 59002924.25436937 }, "16.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 3825911.462200029, "T_cal": 3825911.462200029 }, "16.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 96333674.7713685, "T_cal": 96333674.7713685 }, "16.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 100733014.13690081, "T_cal": 100733014.13690081 }, "16.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 14001185.283757765, "T_cal": 14001185.283757765 }, "17.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 90999721.37062609, "T_cal": 90999721.37062609 }, "17.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 94144494.36542645, "T_cal": 94144494.36542645 }, "17.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 64403453.469662726, "T_cal": 64403453.469662726 }, "17.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 3025521.39197014, "T_cal": 3025521.39197014 }, "17.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 114185746.6778281, "T_cal": 114185746.6778281 }, "17.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 115542679.64477032, "T_cal": 115542679.64477032 }, "17.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 15649314.247886404, "T_cal": 15649314.247886404 }, "18.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 110103363.06979746, "T_cal": 110103363.06979746 }, "18.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 117367270.214385, "T_cal": 117367270.214385 }, "18.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 82164180.72211114, "T_cal": 82164180.72211114 }, "18.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 3166210.5850431784, "T_cal": 3166210.5850431784 }, "18.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 134126277.65064615, "T_cal": 134126277.65064615 }, "18.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 131203028.10500616, "T_cal": 131203028.10500616 }, "18.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 18484361.330206107, "T_cal": 18484361.330206107 }, "19.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 101768509.07318947, "T_cal": 101768509.07318947 }, "19.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 103998909.26405013, "T_cal": 103998909.26405013 }, "19.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 81145647.71285674, "T_cal": 81145647.71285674 }, "19.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 3372990.023941164, "T_cal": 3372990.023941164 }, "19.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 149760126.86008048, "T_cal": 149760126.86008048 }, "19.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 143932926.40695226, "T_cal": 143932926.40695226 }, "19.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 20783010.553232066, "T_cal": 20783010.553232066 }, "20.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 102824131.11793846, "T_cal": 102824131.11793846 }, "20.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 106997026.2702837, "T_cal": 106997026.2702837 }, "20.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 88466865.76269193, "T_cal": 88466865.76269193 }, "20.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 3906091.420468346, "T_cal": 3906091.420468346 }, "20.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 158846875.585763, "T_cal": 158846875.585763 }, "20.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 150288909.27072227, "T_cal": 150288909.27072227 }, "20.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 25077753.109370977, "T_cal": 25077753.109370977 }, "21.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 117249359.84519279, "T_cal": 117249359.84519279 }, "21.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 121257296.85613456, "T_cal": 121257296.85613456 }, "21.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 104137249.60211886, "T_cal": 104137249.60211886 }, "21.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 3796064.3754783506, "T_cal": 3796064.3754783506 }, "21.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 178408851.44575983, "T_cal": 178408851.44575983 }, "21.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 165147466.36618304, "T_cal": 165147466.36618304 }, "21.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 27767506.54603853, "T_cal": 27767506.54603853 }, "22.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 134961010.2448611, "T_cal": 134961010.2448611 }, "22.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 140511333.76406902, "T_cal": 140511333.76406902 }, "22.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 115200885.73238042, "T_cal": 115200885.73238042 }, "22.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 5330501.721603332, "T_cal": 5330501.721603332 }, "22.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 197246783.0078159, "T_cal": 197246783.0078159 }, "22.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 179827121.52238226, "T_cal": 179827121.52238226 }, "22.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 32466160.77768684, "T_cal": 32466160.77768684 }, "23.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 173956514.88643295, "T_cal": 173956514.88643295 }, "23.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 176127364.40138587, "T_cal": 176127364.40138587 }, "23.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 146970682.6782238, "T_cal": 146970682.6782238 }, "23.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 5986032.564459117, "T_cal": 5986032.564459117 }, "23.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 221781447.51768112, "T_cal": 221781447.51768112 }, "23.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 203256305.40707603, "T_cal": 203256305.40707603 }, "23.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 35625783.40628207, "T_cal": 35625783.40628207 }, "24.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 135040558.8063746, "T_cal": 135040558.8063746 }, "24.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 138802684.4165553, "T_cal": 138802684.4165553 }, "24.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 139502322.0703703, "T_cal": 139502322.0703703 }, "24.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 6309441.988455492, "T_cal": 6309441.988455492 }, "24.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 244754670.52814245, "T_cal": 244754670.52814245 }, "24.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 224345108.1180973, "T_cal": 224345108.1180973 }, "24.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 38812148.47649793, "T_cal": 38812148.47649793 }, "25.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 192676375.26197278, "T_cal": 192676375.26197278 }, "25.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 199974050.3390035, "T_cal": 199974050.3390035 }, "25.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 187314783.59108984, "T_cal": 187314783.59108984 }, "25.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 5900984.924451758, "T_cal": 5900984.924451758 }, "25.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 267870776.95862532, "T_cal": 267870776.95862532 }, "25.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 245707416.92936134, "T_cal": 245707416.92936134 }, "25.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 41059274.86111863, "T_cal": 41059274.86111863 }, "26.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 159876587.18329144, "T_cal": 159876587.18329144 }, "26.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 167987712.37976366, "T_cal": 167987712.37976366 }, "26.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 178547411.6502079, "T_cal": 178547411.6502079 }, "26.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 7623597.176070761, "T_cal": 7623597.176070761 }, "26.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 294025002.11450833, "T_cal": 294025002.11450833 }, "26.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 270022573.74512374, "T_cal": 270022573.74512374 }, "26.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 45782097.51798873, "T_cal": 45782097.51798873 }, "27.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 217612197.10079458, "T_cal": 217612197.10079458 }, "27.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 228326841.0355397, "T_cal": 228326841.0355397 }, "27.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 203613303.3052567, "T_cal": 203613303.3052567 }, "27.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 9217756.436816841, "T_cal": 9217756.436816841 }, "27.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 322599853.83160543, "T_cal": 322599853.83160543 }, "27.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 297211624.6158835, "T_cal": 297211624.6158835 }, "27.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 52167899.43894178, "T_cal": 52167899.43894178 }, "28.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 200838248.56867433, "T_cal": 200838248.56867433 }, "28.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 214336534.23926508, "T_cal": 214336534.23926508 }, "28.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 222512980.9407476, "T_cal": 222512980.9407476 }, "28.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 9593215.460344194, "T_cal": 9593215.460344194 }, "28.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 332713275.77167547, "T_cal": 332713275.77167547 }, "28.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 314053516.85497355, "T_cal": 314053516.85497355 }, "28.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 61121425.034155086, "T_cal": 61121425.034155086 }, "29.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 157059065.2338967, "T_cal": 157059065.2338967 }, "29.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 165251066.88863057, "T_cal": 165251066.88863057 }, "29.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 205795377.7990726, "T_cal": 205795377.7990726 }, "29.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 11326351.557557376, "T_cal": 11326351.557557376 }, "29.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 343514311.44628775, "T_cal": 343514311.44628775 }, "29.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 325287679.0273143, "T_cal": 325287679.0273143 }, "29.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 70370844.80613631, "T_cal": 70370844.80613631 }, "30.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 187904103.74662805, "T_cal": 187904103.74662805 }, "30.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 198933376.01896542, "T_cal": 198933376.01896542 }, "30.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 233478558.48754305, "T_cal": 233478558.48754305 }, "30.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 11472074.066494709, "T_cal": 11472074.066494709 }, "30.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 343689739.0538466, "T_cal": 343689739.0538466 }, "30.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 316121128.9378124, "T_cal": 316121128.9378124 }, "30.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 81170606.37089503, "T_cal": 81170606.37089503 }, "31.q_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 99208584.89226724, "T_cal": 99208584.89226724 }, "31.k_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 110916878.23399386, "T_cal": 110916878.23399386 }, "31.v_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 121095456.31283128, "T_cal": 121095456.31283128 }, "31.o_proj": { "r": 1024, "k_u": 1024, "k_s": 0, "T_cal_prefix": 10255140.933228899, "T_cal": 10255140.933228899 }, "31.gate_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 275096568.15761864, "T_cal": 275096568.15761864 }, "31.up_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 246626847.1224712, "T_cal": 246626847.1224712 }, "31.down_proj": { "r": 1492, "k_u": 1492, "k_s": 0, "T_cal_prefix": 91462677.79018125, "T_cal": 91462677.79018125 } }, "ranks": { "0.q_proj": 1024, "0.k_proj": 1024, "0.v_proj": 1024, "0.o_proj": 1024, "0.gate_proj": 1492, "0.up_proj": 1492, "0.down_proj": 1492, "1.q_proj": 1024, "1.k_proj": 1024, "1.v_proj": 1024, "1.o_proj": 1024, "1.gate_proj": 1492, "1.up_proj": 1492, "1.down_proj": 1492, "2.q_proj": 1024, "2.k_proj": 1024, "2.v_proj": 1024, "2.o_proj": 1024, "2.gate_proj": 1492, "2.up_proj": 1492, "2.down_proj": 1492, "3.q_proj": 1024, "3.k_proj": 1024, "3.v_proj": 1024, "3.o_proj": 1024, "3.gate_proj": 1492, "3.up_proj": 1492, "3.down_proj": 1492, "4.q_proj": 1024, "4.k_proj": 1024, "4.v_proj": 1024, "4.o_proj": 1024, "4.gate_proj": 1492, "4.up_proj": 1492, "4.down_proj": 1492, "5.q_proj": 1024, "5.k_proj": 1024, "5.v_proj": 1024, "5.o_proj": 1024, "5.gate_proj": 1492, "5.up_proj": 1492, "5.down_proj": 1492, "6.q_proj": 1024, "6.k_proj": 1024, "6.v_proj": 1024, "6.o_proj": 1024, "6.gate_proj": 1492, "6.up_proj": 1492, "6.down_proj": 1492, "7.q_proj": 1024, "7.k_proj": 1024, "7.v_proj": 1024, "7.o_proj": 1024, "7.gate_proj": 1492, "7.up_proj": 1492, "7.down_proj": 1492, "8.q_proj": 1024, "8.k_proj": 1024, "8.v_proj": 1024, "8.o_proj": 1024, "8.gate_proj": 1492, "8.up_proj": 1492, "8.down_proj": 1492, "9.q_proj": 1024, "9.k_proj": 1024, "9.v_proj": 1024, "9.o_proj": 1024, "9.gate_proj": 1492, "9.up_proj": 1492, "9.down_proj": 1492, "10.q_proj": 1024, "10.k_proj": 1024, "10.v_proj": 1024, "10.o_proj": 1024, "10.gate_proj": 1492, "10.up_proj": 1492, "10.down_proj": 1492, "11.q_proj": 1024, "11.k_proj": 1024, "11.v_proj": 1024, "11.o_proj": 1024, "11.gate_proj": 1492, "11.up_proj": 1492, "11.down_proj": 1492, "12.q_proj": 1024, "12.k_proj": 1024, "12.v_proj": 1024, "12.o_proj": 1024, "12.gate_proj": 1492, "12.up_proj": 1492, "12.down_proj": 1492, "13.q_proj": 1024, "13.k_proj": 1024, "13.v_proj": 1024, "13.o_proj": 1024, "13.gate_proj": 1492, "13.up_proj": 1492, "13.down_proj": 1492, "14.q_proj": 1024, "14.k_proj": 1024, "14.v_proj": 1024, "14.o_proj": 1024, "14.gate_proj": 1492, "14.up_proj": 1492, "14.down_proj": 1492, "15.q_proj": 1024, "15.k_proj": 1024, "15.v_proj": 1024, "15.o_proj": 1024, "15.gate_proj": 1492, "15.up_proj": 1492, "15.down_proj": 1492, "16.q_proj": 1024, "16.k_proj": 1024, "16.v_proj": 1024, "16.o_proj": 1024, "16.gate_proj": 1492, "16.up_proj": 1492, "16.down_proj": 1492, "17.q_proj": 1024, "17.k_proj": 1024, "17.v_proj": 1024, "17.o_proj": 1024, "17.gate_proj": 1492, "17.up_proj": 1492, "17.down_proj": 1492, "18.q_proj": 1024, "18.k_proj": 1024, "18.v_proj": 1024, "18.o_proj": 1024, "18.gate_proj": 1492, "18.up_proj": 1492, "18.down_proj": 1492, "19.q_proj": 1024, "19.k_proj": 1024, "19.v_proj": 1024, "19.o_proj": 1024, "19.gate_proj": 1492, "19.up_proj": 1492, "19.down_proj": 1492, "20.q_proj": 1024, "20.k_proj": 1024, "20.v_proj": 1024, "20.o_proj": 1024, "20.gate_proj": 1492, "20.up_proj": 1492, "20.down_proj": 1492, "21.q_proj": 1024, "21.k_proj": 1024, "21.v_proj": 1024, "21.o_proj": 1024, "21.gate_proj": 1492, "21.up_proj": 1492, "21.down_proj": 1492, "22.q_proj": 1024, "22.k_proj": 1024, "22.v_proj": 1024, "22.o_proj": 1024, "22.gate_proj": 1492, "22.up_proj": 1492, "22.down_proj": 1492, "23.q_proj": 1024, "23.k_proj": 1024, "23.v_proj": 1024, "23.o_proj": 1024, "23.gate_proj": 1492, "23.up_proj": 1492, "23.down_proj": 1492, "24.q_proj": 1024, "24.k_proj": 1024, "24.v_proj": 1024, "24.o_proj": 1024, "24.gate_proj": 1492, "24.up_proj": 1492, "24.down_proj": 1492, "25.q_proj": 1024, "25.k_proj": 1024, "25.v_proj": 1024, "25.o_proj": 1024, "25.gate_proj": 1492, "25.up_proj": 1492, "25.down_proj": 1492, "26.q_proj": 1024, "26.k_proj": 1024, "26.v_proj": 1024, "26.o_proj": 1024, "26.gate_proj": 1492, "26.up_proj": 1492, "26.down_proj": 1492, "27.q_proj": 1024, "27.k_proj": 1024, "27.v_proj": 1024, "27.o_proj": 1024, "27.gate_proj": 1492, "27.up_proj": 1492, "27.down_proj": 1492, "28.q_proj": 1024, "28.k_proj": 1024, "28.v_proj": 1024, "28.o_proj": 1024, "28.gate_proj": 1492, "28.up_proj": 1492, "28.down_proj": 1492, "29.q_proj": 1024, "29.k_proj": 1024, "29.v_proj": 1024, "29.o_proj": 1024, "29.gate_proj": 1492, "29.up_proj": 1492, "29.down_proj": 1492, "30.q_proj": 1024, "30.k_proj": 1024, "30.v_proj": 1024, "30.o_proj": 1024, "30.gate_proj": 1492, "30.up_proj": 1492, "30.down_proj": 1492, "31.q_proj": 1024, "31.k_proj": 1024, "31.v_proj": 1024, "31.o_proj": 1024, "31.gate_proj": 1492, "31.up_proj": 1492, "31.down_proj": 1492 }, "factored": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove50/truncated/factored.pt", "folded": false, "qkv_bias_restored": 0 }