{ "source": "/lustre/jeesup0103/svd-safety/assets/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590", "upstream": "AIoT-MLSys-Lab/SVD-LLM", "step": "whiten", "ratio_removed": 0.1, "internal_ratio_kept": 0.9, "nsamples": 256, "seed": 42, "calib_spec": "wikitext2:256", "calib": [ { "name": "wikitext2", "n": 255, "source": "upstream get_calib_train_data" } ], "n_calib_sequences": 255, "rank_by": "resmix", "whitening": { "lam": 0.0, "norm": "token", "ridge": 0.0, "token_mask": "nospecial", "utility_cache": "/lustre/jeesup0103/svd-safety/work/whiten_cache/utility_wikitext256", "utility_calib": "wikitext2:256", "utility_tokens": 522240 }, "whiten_lam": 0.0, "lam": 0.0, "fisher_scores": null, "fisher_num": null, "fisher_den": null, "fisher_form": null, "fisher_control": null, "fisher_floor": null, "safety_calib": null, "resmix_k": 0.0, "resmix_k_abs": null, "resmix_alpha": 1.0, "n_safety_tokens": null, "ranking": { "n_matrices": 224, "alpha": 1.0, "k_s_total": 0, "k_s_frac_of_rank": 0.0, "utility_error_ratio": 1.0, "resmix_k": 0.0, "resmix_k_abs": null, "resmix_alpha": 1.0 }, "ranking_per_matrix": { "0.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 10.785678176430773, "T_cal": 10.785678176430773 }, "0.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 9.576390170908583, "T_cal": 9.576390170908583 }, "0.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 851.6657157551547, "T_cal": 851.6657157551547 }, "0.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 0.9780601726363409, "T_cal": 0.9780601726363409 }, "0.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 89500.30917201788, "T_cal": 89500.30917201788 }, "0.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 92107.91324240528, "T_cal": 92107.91324240528 }, "0.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 2099.17623664463, "T_cal": 2099.17623664463 }, "1.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 8719.575457896477, "T_cal": 8719.575457896477 }, "1.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 6348.398214962525, "T_cal": 6348.398214962525 }, "1.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 22890.31522699863, "T_cal": 22890.31522699863 }, "1.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 98.1451973636449, "T_cal": 98.1451973636449 }, "1.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 329123.88260184403, "T_cal": 329123.88260184403 }, "1.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 376866.56578857085, "T_cal": 376866.56578857085 }, "1.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 28736.74082056575, "T_cal": 28736.74082056575 }, "2.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 643893.6614395604, "T_cal": 643893.6614395604 }, "2.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 626112.885156469, "T_cal": 626112.885156469 }, "2.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 876510.7086711043, "T_cal": 876510.7086711043 }, "2.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 10899.537467294535, "T_cal": 10899.537467294535 }, "2.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 1693703.440605626, "T_cal": 1693703.440605626 }, "2.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 1696804.2039791269, "T_cal": 1696804.2039791269 }, "2.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 85977.69509733199, "T_cal": 85977.69509733199 }, "3.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 5783209.987075051, "T_cal": 5783209.987075051 }, "3.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 6152009.641441202, "T_cal": 6152009.641441202 }, "3.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 3739286.972493657, "T_cal": 3739286.972493657 }, "3.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 19633.092639652994, "T_cal": 19633.092639652994 }, "3.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 5126539.8926883275, "T_cal": 5126539.8926883275 }, "3.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 4762765.7752103135, "T_cal": 4762765.7752103135 }, "3.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 151902.5633119275, "T_cal": 151902.5633119275 }, "4.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 5857197.810485085, "T_cal": 5857197.810485085 }, "4.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 6178465.372496818, "T_cal": 6178465.372496818 }, "4.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 3967419.9790056134, "T_cal": 3967419.9790056134 }, "4.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 44046.95099997806, "T_cal": 44046.95099997806 }, "4.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 7840698.699492722, "T_cal": 7840698.699492722 }, "4.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 7040923.029561698, "T_cal": 7040923.029561698 }, "4.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 280243.8009020678, "T_cal": 280243.8009020678 }, "5.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 7545364.521031534, "T_cal": 7545364.521031534 }, "5.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 8063765.271419729, "T_cal": 8063765.271419729 }, "5.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 4943144.505550002, "T_cal": 4943144.505550002 }, "5.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 58175.060551189556, "T_cal": 58175.060551189556 }, "5.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 9873067.979936454, "T_cal": 9873067.979936454 }, "5.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 8941202.13321339, "T_cal": 8941202.13321339 }, "5.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 424218.746589352, "T_cal": 424218.746589352 }, "6.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 10533861.771791458, "T_cal": 10533861.771791458 }, "6.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 11390276.337218832, "T_cal": 11390276.337218832 }, "6.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 6543838.750572076, "T_cal": 6543838.750572076 }, "6.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 103800.18289654578, "T_cal": 103800.18289654578 }, "6.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 12771164.726543933, "T_cal": 12771164.726543933 }, "6.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 11397835.174568733, "T_cal": 11397835.174568733 }, "6.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 604886.0140598821, "T_cal": 604886.0140598821 }, "7.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 11844207.879055038, "T_cal": 11844207.879055038 }, "7.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 12400266.770684075, "T_cal": 12400266.770684075 }, "7.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 7136304.962956544, "T_cal": 7136304.962956544 }, "7.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 148138.74464088996, "T_cal": 148138.74464088996 }, "7.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 15039607.582688756, "T_cal": 15039607.582688756 }, "7.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 13674887.732154544, "T_cal": 13674887.732154544 }, "7.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 803697.9954367423, "T_cal": 803697.9954367423 }, "8.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 12709352.285616899, "T_cal": 12709352.285616899 }, "8.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 13722898.40869642, "T_cal": 13722898.40869642 }, "8.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 7054155.237657632, "T_cal": 7054155.237657632 }, "8.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 220494.06387088224, "T_cal": 220494.06387088224 }, "8.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 15542306.874569666, "T_cal": 15542306.874569666 }, "8.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 14794875.010815449, "T_cal": 14794875.010815449 }, "8.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 982559.5498754395, "T_cal": 982559.5498754395 }, "9.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 15184121.982329842, "T_cal": 15184121.982329842 }, "9.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 16383592.258965734, "T_cal": 16383592.258965734 }, "9.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 7869652.540471686, "T_cal": 7869652.540471686 }, "9.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 326461.51250768616, "T_cal": 326461.51250768616 }, "9.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 16271731.347973311, "T_cal": 16271731.347973311 }, "9.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 15953925.992089037, "T_cal": 15953925.992089037 }, "9.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 1187740.602015712, "T_cal": 1187740.602015712 }, "10.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 13435789.945286755, "T_cal": 13435789.945286755 }, "10.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 15122004.14128048, "T_cal": 15122004.14128048 }, "10.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 7834700.253071317, "T_cal": 7834700.253071317 }, "10.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 411647.8105691812, "T_cal": 411647.8105691812 }, "10.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 16575456.584854057, "T_cal": 16575456.584854057 }, "10.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 16757814.787526622, "T_cal": 16757814.787526622 }, "10.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 1316334.097178603, "T_cal": 1316334.097178603 }, "11.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 13930622.718331898, "T_cal": 13930622.718331898 }, "11.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 15007569.816916827, "T_cal": 15007569.816916827 }, "11.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 9711772.716010273, "T_cal": 9711772.716010273 }, "11.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 391418.17282386805, "T_cal": 391418.17282386805 }, "11.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 18402714.369628422, "T_cal": 18402714.369628422 }, "11.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 19088554.366343856, "T_cal": 19088554.366343856 }, "11.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 1495485.0911699792, "T_cal": 1495485.0911699792 }, "12.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 18748398.866943624, "T_cal": 18748398.866943624 }, "12.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 19914010.67244915, "T_cal": 19914010.67244915 }, "12.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 10801728.444651768, "T_cal": 10801728.444651768 }, "12.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 531758.5048293495, "T_cal": 531758.5048293495 }, "12.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 19764191.62510787, "T_cal": 19764191.62510787 }, "12.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 20794292.418899104, "T_cal": 20794292.418899104 }, "12.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 1736375.6087208553, "T_cal": 1736375.6087208553 }, "13.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 20292510.921908595, "T_cal": 20292510.921908595 }, "13.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 22995179.546872757, "T_cal": 22995179.546872757 }, "13.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 14077361.372142034, "T_cal": 14077361.372142034 }, "13.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 540804.128938798, "T_cal": 540804.128938798 }, "13.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 20812667.53851728, "T_cal": 20812667.53851728 }, "13.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 22549834.16205047, "T_cal": 22549834.16205047 }, "13.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 2137573.7105607325, "T_cal": 2137573.7105607325 }, "14.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 18825714.543996416, "T_cal": 18825714.543996416 }, "14.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 20136168.07042644, "T_cal": 20136168.07042644 }, "14.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 13359328.927838562, "T_cal": 13359328.927838562 }, "14.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 689326.1516168148, "T_cal": 689326.1516168148 }, "14.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 22786515.866438046, "T_cal": 22786515.866438046 }, "14.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 24728059.81873543, "T_cal": 24728059.81873543 }, "14.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 2517011.435474057, "T_cal": 2517011.435474057 }, "15.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 19993806.91956532, "T_cal": 19993806.91956532 }, "15.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 21747526.803754296, "T_cal": 21747526.803754296 }, "15.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 14665061.06597065, "T_cal": 14665061.06597065 }, "15.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 817638.8098200579, "T_cal": 817638.8098200579 }, "15.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 24911255.812186185, "T_cal": 24911255.812186185 }, "15.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 27084609.74382753, "T_cal": 27084609.74382753 }, "15.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 3193436.979680022, "T_cal": 3193436.979680022 }, "16.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 20407912.52514683, "T_cal": 20407912.52514683 }, "16.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 21974787.172197178, "T_cal": 21974787.172197178 }, "16.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 18757995.0596334, "T_cal": 18757995.0596334 }, "16.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 1071887.20644118, "T_cal": 1071887.20644118 }, "16.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 26989833.561512306, "T_cal": 26989833.561512306 }, "16.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 28729017.121802475, "T_cal": 28729017.121802475 }, "16.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 4155374.8409035476, "T_cal": 4155374.8409035476 }, "17.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 25441887.973660365, "T_cal": 25441887.973660365 }, "17.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 26466781.651458476, "T_cal": 26466781.651458476 }, "17.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 20486151.87593612, "T_cal": 20486151.87593612 }, "17.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 922322.9825638515, "T_cal": 922322.9825638515 }, "17.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 32072077.254918516, "T_cal": 32072077.254918516 }, "17.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 32978860.75940479, "T_cal": 32978860.75940479 }, "17.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 4637992.671098169, "T_cal": 4637992.671098169 }, "18.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 31773729.391317233, "T_cal": 31773729.391317233 }, "18.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 34017678.001551084, "T_cal": 34017678.001551084 }, "18.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 26689941.41991458, "T_cal": 26689941.41991458 }, "18.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 938748.9824173935, "T_cal": 938748.9824173935 }, "18.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 37749984.50054178, "T_cal": 37749984.50054178 }, "18.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 37563549.93561437, "T_cal": 37563549.93561437 }, "18.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 5552828.787711998, "T_cal": 5552828.787711998 }, "19.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 28647353.80244772, "T_cal": 28647353.80244772 }, "19.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 29289822.33732438, "T_cal": 29289822.33732438 }, "19.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 25667160.26837124, "T_cal": 25667160.26837124 }, "19.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 1032724.8317759677, "T_cal": 1032724.8317759677 }, "19.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 42595598.11038089, "T_cal": 42595598.11038089 }, "19.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 41513706.856649965, "T_cal": 41513706.856649965 }, "19.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 6309925.385078111, "T_cal": 6309925.385078111 }, "20.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 28436851.374570876, "T_cal": 28436851.374570876 }, "20.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 29759141.79584817, "T_cal": 29759141.79584817 }, "20.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 28791771.883710474, "T_cal": 28791771.883710474 }, "20.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 1204475.571156554, "T_cal": 1204475.571156554 }, "20.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 45059279.89659271, "T_cal": 45059279.89659271 }, "20.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 43208565.658598244, "T_cal": 43208565.658598244 }, "20.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 7628871.560222485, "T_cal": 7628871.560222485 }, "21.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 32768398.130046345, "T_cal": 32768398.130046345 }, "21.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 33770010.93750383, "T_cal": 33770010.93750383 }, "21.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 33815038.04286927, "T_cal": 33815038.04286927 }, "21.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 1186598.6591508216, "T_cal": 1186598.6591508216 }, "21.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 50949707.43289541, "T_cal": 50949707.43289541 }, "21.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 47731695.09471324, "T_cal": 47731695.09471324 }, "21.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 8489349.957435494, "T_cal": 8489349.957435494 }, "22.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 38265573.64003351, "T_cal": 38265573.64003351 }, "22.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 40043333.69933437, "T_cal": 40043333.69933437 }, "22.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 37850521.35180683, "T_cal": 37850521.35180683 }, "22.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 1687446.2246034364, "T_cal": 1687446.2246034364 }, "22.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 56684983.32320996, "T_cal": 56684983.32320996 }, "22.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 52258393.50257513, "T_cal": 52258393.50257513 }, "22.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 9890840.02519774, "T_cal": 9890840.02519774 }, "23.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 51741264.84595457, "T_cal": 51741264.84595457 }, "23.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 52444286.27460586, "T_cal": 52444286.27460586 }, "23.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 48520675.52845783, "T_cal": 48520675.52845783 }, "23.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 1967040.5017786198, "T_cal": 1967040.5017786198 }, "23.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 64495712.7115826, "T_cal": 64495712.7115826 }, "23.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 59670810.45598352, "T_cal": 59670810.45598352 }, "23.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 10872405.012488443, "T_cal": 10872405.012488443 }, "24.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 37388957.17629345, "T_cal": 37388957.17629345 }, "24.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 38682147.232396245, "T_cal": 38682147.232396245 }, "24.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 46526989.69213562, "T_cal": 46526989.69213562 }, "24.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 2045753.1178002125, "T_cal": 2045753.1178002125 }, "24.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 71713721.48563766, "T_cal": 71713721.48563766 }, "24.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 66417296.29878324, "T_cal": 66417296.29878324 }, "24.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 11858175.648777504, "T_cal": 11858175.648777504 }, "25.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 58150761.39573605, "T_cal": 58150761.39573605 }, "25.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 60666231.365755424, "T_cal": 60666231.365755424 }, "25.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 63400534.17654945, "T_cal": 63400534.17654945 }, "25.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 1833248.6132108124, "T_cal": 1833248.6132108124 }, "25.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 79249577.4974488, "T_cal": 79249577.4974488 }, "25.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 73455470.56279409, "T_cal": 73455470.56279409 }, "25.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 12594951.21631376, "T_cal": 12594951.21631376 }, "26.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 46642155.81444648, "T_cal": 46642155.81444648 }, "26.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 49238810.41841708, "T_cal": 49238810.41841708 }, "26.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 60145957.693433836, "T_cal": 60145957.693433836 }, "26.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 2269621.8077429, "T_cal": 2269621.8077429 }, "26.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 87286412.45394044, "T_cal": 87286412.45394044 }, "26.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 81332614.31588593, "T_cal": 81332614.31588593 }, "26.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 14127550.918734217, "T_cal": 14127550.918734217 }, "27.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 69267576.00702935, "T_cal": 69267576.00702935 }, "27.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 73067124.26488246, "T_cal": 73067124.26488246 }, "27.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 69976263.48066255, "T_cal": 69976263.48066255 }, "27.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 2894573.086826196, "T_cal": 2894573.086826196 }, "27.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 96620978.22339875, "T_cal": 96620978.22339875 }, "27.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 90349637.19218573, "T_cal": 90349637.19218573 }, "27.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 16197426.509603195, "T_cal": 16197426.509603195 }, "28.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 62137063.336638995, "T_cal": 62137063.336638995 }, "28.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 66669531.8016634, "T_cal": 66669531.8016634 }, "28.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 76545127.1888986, "T_cal": 76545127.1888986 }, "28.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 2922378.032600498, "T_cal": 2922378.032600498 }, "28.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 100409398.39371276, "T_cal": 100409398.39371276 }, "28.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 96159260.31840637, "T_cal": 96159260.31840637 }, "28.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 19153271.810673572, "T_cal": 19153271.810673572 }, "29.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 46767363.022601195, "T_cal": 46767363.022601195 }, "29.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 49546393.80525367, "T_cal": 49546393.80525367 }, "29.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 69785930.51983362, "T_cal": 69785930.51983362 }, "29.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 3541892.7473701574, "T_cal": 3541892.7473701574 }, "29.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 103934394.06969668, "T_cal": 103934394.06969668 }, "29.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 99789746.97699946, "T_cal": 99789746.97699946 }, "29.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 22192004.713112906, "T_cal": 22192004.713112906 }, "30.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 59149532.14206349, "T_cal": 59149532.14206349 }, "30.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 62872528.52602811, "T_cal": 62872528.52602811 }, "30.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 80734685.89489755, "T_cal": 80734685.89489755 }, "30.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 3551552.105358068, "T_cal": 3551552.105358068 }, "30.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 103809427.73050085, "T_cal": 103809427.73050085 }, "30.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 96934270.2299757, "T_cal": 96934270.2299757 }, "30.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 25570850.722484395, "T_cal": 25570850.722484395 }, "31.q_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 29581901.946942493, "T_cal": 29581901.946942493 }, "31.k_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 33079206.25391892, "T_cal": 33079206.25391892 }, "31.v_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 40068947.4937045, "T_cal": 40068947.4937045 }, "31.o_proj": { "r": 1843, "k_u": 1843, "k_s": 0, "T_cal_prefix": 3047059.923759697, "T_cal": 3047059.923759697 }, "31.gate_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 81595876.46882962, "T_cal": 81595876.46882962 }, "31.up_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 74195895.02706979, "T_cal": 74195895.02706979 }, "31.down_proj": { "r": 2686, "k_u": 2686, "k_s": 0, "T_cal_prefix": 27905283.335053638, "T_cal": 27905283.335053638 } }, "ranks": { "0.q_proj": 1843, "0.k_proj": 1843, "0.v_proj": 1843, "0.o_proj": 1843, "0.gate_proj": 2686, "0.up_proj": 2686, "0.down_proj": 2686, "1.q_proj": 1843, "1.k_proj": 1843, "1.v_proj": 1843, "1.o_proj": 1843, "1.gate_proj": 2686, "1.up_proj": 2686, "1.down_proj": 2686, "2.q_proj": 1843, "2.k_proj": 1843, "2.v_proj": 1843, "2.o_proj": 1843, "2.gate_proj": 2686, "2.up_proj": 2686, "2.down_proj": 2686, "3.q_proj": 1843, "3.k_proj": 1843, "3.v_proj": 1843, "3.o_proj": 1843, "3.gate_proj": 2686, "3.up_proj": 2686, "3.down_proj": 2686, "4.q_proj": 1843, "4.k_proj": 1843, "4.v_proj": 1843, "4.o_proj": 1843, "4.gate_proj": 2686, "4.up_proj": 2686, "4.down_proj": 2686, "5.q_proj": 1843, "5.k_proj": 1843, "5.v_proj": 1843, "5.o_proj": 1843, "5.gate_proj": 2686, "5.up_proj": 2686, "5.down_proj": 2686, "6.q_proj": 1843, "6.k_proj": 1843, "6.v_proj": 1843, "6.o_proj": 1843, "6.gate_proj": 2686, "6.up_proj": 2686, "6.down_proj": 2686, "7.q_proj": 1843, "7.k_proj": 1843, "7.v_proj": 1843, "7.o_proj": 1843, "7.gate_proj": 2686, "7.up_proj": 2686, "7.down_proj": 2686, "8.q_proj": 1843, "8.k_proj": 1843, "8.v_proj": 1843, "8.o_proj": 1843, "8.gate_proj": 2686, "8.up_proj": 2686, "8.down_proj": 2686, "9.q_proj": 1843, "9.k_proj": 1843, "9.v_proj": 1843, "9.o_proj": 1843, "9.gate_proj": 2686, "9.up_proj": 2686, "9.down_proj": 2686, "10.q_proj": 1843, "10.k_proj": 1843, "10.v_proj": 1843, "10.o_proj": 1843, "10.gate_proj": 2686, "10.up_proj": 2686, "10.down_proj": 2686, "11.q_proj": 1843, "11.k_proj": 1843, "11.v_proj": 1843, "11.o_proj": 1843, "11.gate_proj": 2686, "11.up_proj": 2686, "11.down_proj": 2686, "12.q_proj": 1843, "12.k_proj": 1843, "12.v_proj": 1843, "12.o_proj": 1843, "12.gate_proj": 2686, "12.up_proj": 2686, "12.down_proj": 2686, "13.q_proj": 1843, "13.k_proj": 1843, "13.v_proj": 1843, "13.o_proj": 1843, "13.gate_proj": 2686, "13.up_proj": 2686, "13.down_proj": 2686, "14.q_proj": 1843, "14.k_proj": 1843, "14.v_proj": 1843, "14.o_proj": 1843, "14.gate_proj": 2686, "14.up_proj": 2686, "14.down_proj": 2686, "15.q_proj": 1843, "15.k_proj": 1843, "15.v_proj": 1843, "15.o_proj": 1843, "15.gate_proj": 2686, "15.up_proj": 2686, "15.down_proj": 2686, "16.q_proj": 1843, "16.k_proj": 1843, "16.v_proj": 1843, "16.o_proj": 1843, "16.gate_proj": 2686, "16.up_proj": 2686, "16.down_proj": 2686, "17.q_proj": 1843, "17.k_proj": 1843, "17.v_proj": 1843, "17.o_proj": 1843, "17.gate_proj": 2686, "17.up_proj": 2686, "17.down_proj": 2686, "18.q_proj": 1843, "18.k_proj": 1843, "18.v_proj": 1843, "18.o_proj": 1843, "18.gate_proj": 2686, "18.up_proj": 2686, "18.down_proj": 2686, "19.q_proj": 1843, "19.k_proj": 1843, "19.v_proj": 1843, "19.o_proj": 1843, "19.gate_proj": 2686, "19.up_proj": 2686, "19.down_proj": 2686, "20.q_proj": 1843, "20.k_proj": 1843, "20.v_proj": 1843, "20.o_proj": 1843, "20.gate_proj": 2686, "20.up_proj": 2686, "20.down_proj": 2686, "21.q_proj": 1843, "21.k_proj": 1843, "21.v_proj": 1843, "21.o_proj": 1843, "21.gate_proj": 2686, "21.up_proj": 2686, "21.down_proj": 2686, "22.q_proj": 1843, "22.k_proj": 1843, "22.v_proj": 1843, "22.o_proj": 1843, "22.gate_proj": 2686, "22.up_proj": 2686, "22.down_proj": 2686, "23.q_proj": 1843, "23.k_proj": 1843, "23.v_proj": 1843, "23.o_proj": 1843, "23.gate_proj": 2686, "23.up_proj": 2686, "23.down_proj": 2686, "24.q_proj": 1843, "24.k_proj": 1843, "24.v_proj": 1843, "24.o_proj": 1843, "24.gate_proj": 2686, "24.up_proj": 2686, "24.down_proj": 2686, "25.q_proj": 1843, "25.k_proj": 1843, "25.v_proj": 1843, "25.o_proj": 1843, "25.gate_proj": 2686, "25.up_proj": 2686, "25.down_proj": 2686, "26.q_proj": 1843, "26.k_proj": 1843, "26.v_proj": 1843, "26.o_proj": 1843, "26.gate_proj": 2686, "26.up_proj": 2686, "26.down_proj": 2686, "27.q_proj": 1843, "27.k_proj": 1843, "27.v_proj": 1843, "27.o_proj": 1843, "27.gate_proj": 2686, "27.up_proj": 2686, "27.down_proj": 2686, "28.q_proj": 1843, "28.k_proj": 1843, "28.v_proj": 1843, "28.o_proj": 1843, "28.gate_proj": 2686, "28.up_proj": 2686, "28.down_proj": 2686, "29.q_proj": 1843, "29.k_proj": 1843, "29.v_proj": 1843, "29.o_proj": 1843, "29.gate_proj": 2686, "29.up_proj": 2686, "29.down_proj": 2686, "30.q_proj": 1843, "30.k_proj": 1843, "30.v_proj": 1843, "30.o_proj": 1843, "30.gate_proj": 2686, "30.up_proj": 2686, "30.down_proj": 2686, "31.q_proj": 1843, "31.k_proj": 1843, "31.v_proj": 1843, "31.o_proj": 1843, "31.gate_proj": 2686, "31.up_proj": 2686, "31.down_proj": 2686 }, "factored": "/lustre/jeesup0103/svd-safety/results/resmix/full/l2_base_k0_a1p0_free_remove10/truncated/factored.pt", "folded": false, "qkv_bias_restored": 0 }