Jeesup/svdsafety_l2_remove50_whiten_protk8
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: Jeesup/svdsafety_l2_remove50_whiten_protk8
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-09-28 08:13:18 +08:00
commit c7f4350fca
15 changed files with 278767 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

950
compression.json Normal file
View File

@@ -0,0 +1,950 @@
{
"source": "assets/models/Llama-2-7b-chat-hf",
"mode": "whiten",
"ratio": 0.5,
"n_calib": 256,
"seqlen": 2048,
"seed": 42,
"mean_rel_err": 0.5757304649700278,
"max_rel_err": 0.6879633665084839,
"ranks": {
"0.q_proj": 1024,
"0.k_proj": 1024,
"0.v_proj": 1024,
"0.o_proj": 1024,
"0.gate_proj": 1492,
"0.up_proj": 1492,
"0.down_proj": 1492,
"1.q_proj": 1024,
"1.k_proj": 1024,
"1.v_proj": 1024,
"1.o_proj": 1024,
"1.gate_proj": 1492,
"1.up_proj": 1492,
"1.down_proj": 1492,
"2.q_proj": 1024,
"2.k_proj": 1024,
"2.v_proj": 1024,
"2.o_proj": 1024,
"2.gate_proj": 1492,
"2.up_proj": 1492,
"2.down_proj": 1492,
"3.q_proj": 1024,
"3.k_proj": 1024,
"3.v_proj": 1024,
"3.o_proj": 1024,
"3.gate_proj": 1492,
"3.up_proj": 1492,
"3.down_proj": 1492,
"4.q_proj": 1024,
"4.k_proj": 1024,
"4.v_proj": 1024,
"4.o_proj": 1024,
"4.gate_proj": 1492,
"4.up_proj": 1492,
"4.down_proj": 1492,
"5.q_proj": 1024,
"5.k_proj": 1024,
"5.v_proj": 1024,
"5.o_proj": 1024,
"5.gate_proj": 1492,
"5.up_proj": 1492,
"5.down_proj": 1492,
"6.q_proj": 1024,
"6.k_proj": 1024,
"6.v_proj": 1024,
"6.o_proj": 1024,
"6.gate_proj": 1492,
"6.up_proj": 1492,
"6.down_proj": 1492,
"7.q_proj": 1024,
"7.k_proj": 1024,
"7.v_proj": 1024,
"7.o_proj": 1024,
"7.gate_proj": 1492,
"7.up_proj": 1492,
"7.down_proj": 1492,
"8.q_proj": 1024,
"8.k_proj": 1024,
"8.v_proj": 1024,
"8.o_proj": 1024,
"8.gate_proj": 1492,
"8.up_proj": 1492,
"8.down_proj": 1492,
"9.q_proj": 1024,
"9.k_proj": 1024,
"9.v_proj": 1024,
"9.o_proj": 1024,
"9.gate_proj": 1492,
"9.up_proj": 1492,
"9.down_proj": 1492,
"10.q_proj": 1024,
"10.k_proj": 1024,
"10.v_proj": 1024,
"10.o_proj": 1024,
"10.gate_proj": 1492,
"10.up_proj": 1492,
"10.down_proj": 1492,
"11.q_proj": 1024,
"11.k_proj": 1024,
"11.v_proj": 1024,
"11.o_proj": 1024,
"11.gate_proj": 1492,
"11.up_proj": 1492,
"11.down_proj": 1492,
"12.q_proj": 1024,
"12.k_proj": 1024,
"12.v_proj": 1024,
"12.o_proj": 1024,
"12.gate_proj": 1492,
"12.up_proj": 1492,
"12.down_proj": 1492,
"13.q_proj": 1024,
"13.k_proj": 1024,
"13.v_proj": 1024,
"13.o_proj": 1024,
"13.gate_proj": 1492,
"13.up_proj": 1492,
"13.down_proj": 1492,
"14.q_proj": 1024,
"14.k_proj": 1024,
"14.v_proj": 1024,
"14.o_proj": 1024,
"14.gate_proj": 1492,
"14.up_proj": 1492,
"14.down_proj": 1492,
"15.q_proj": 1024,
"15.k_proj": 1024,
"15.v_proj": 1024,
"15.o_proj": 1024,
"15.gate_proj": 1492,
"15.up_proj": 1492,
"15.down_proj": 1492,
"16.q_proj": 1024,
"16.k_proj": 1024,
"16.v_proj": 1024,
"16.o_proj": 1024,
"16.gate_proj": 1492,
"16.up_proj": 1492,
"16.down_proj": 1492,
"17.q_proj": 1024,
"17.k_proj": 1024,
"17.v_proj": 1024,
"17.o_proj": 1024,
"17.gate_proj": 1492,
"17.up_proj": 1492,
"17.down_proj": 1492,
"18.q_proj": 1024,
"18.k_proj": 1024,
"18.v_proj": 1024,
"18.o_proj": 1024,
"18.gate_proj": 1492,
"18.up_proj": 1492,
"18.down_proj": 1492,
"19.q_proj": 1024,
"19.k_proj": 1024,
"19.v_proj": 1024,
"19.o_proj": 1024,
"19.gate_proj": 1492,
"19.up_proj": 1492,
"19.down_proj": 1492,
"20.q_proj": 1024,
"20.k_proj": 1024,
"20.v_proj": 1024,
"20.o_proj": 1024,
"20.gate_proj": 1492,
"20.up_proj": 1492,
"20.down_proj": 1492,
"21.q_proj": 1024,
"21.k_proj": 1024,
"21.v_proj": 1024,
"21.o_proj": 1024,
"21.gate_proj": 1492,
"21.up_proj": 1492,
"21.down_proj": 1492,
"22.q_proj": 1024,
"22.k_proj": 1024,
"22.v_proj": 1024,
"22.o_proj": 1024,
"22.gate_proj": 1492,
"22.up_proj": 1492,
"22.down_proj": 1492,
"23.q_proj": 1024,
"23.k_proj": 1024,
"23.v_proj": 1024,
"23.o_proj": 1024,
"23.gate_proj": 1492,
"23.up_proj": 1492,
"23.down_proj": 1492,
"24.q_proj": 1024,
"24.k_proj": 1024,
"24.v_proj": 1024,
"24.o_proj": 1024,
"24.gate_proj": 1492,
"24.up_proj": 1492,
"24.down_proj": 1492,
"25.q_proj": 1024,
"25.k_proj": 1024,
"25.v_proj": 1024,
"25.o_proj": 1024,
"25.gate_proj": 1492,
"25.up_proj": 1492,
"25.down_proj": 1492,
"26.q_proj": 1024,
"26.k_proj": 1024,
"26.v_proj": 1024,
"26.o_proj": 1024,
"26.gate_proj": 1492,
"26.up_proj": 1492,
"26.down_proj": 1492,
"27.q_proj": 1024,
"27.k_proj": 1024,
"27.v_proj": 1024,
"27.o_proj": 1024,
"27.gate_proj": 1492,
"27.up_proj": 1492,
"27.down_proj": 1492,
"28.q_proj": 1024,
"28.k_proj": 1024,
"28.v_proj": 1024,
"28.o_proj": 1024,
"28.gate_proj": 1492,
"28.up_proj": 1492,
"28.down_proj": 1492,
"29.q_proj": 1024,
"29.k_proj": 1024,
"29.v_proj": 1024,
"29.o_proj": 1024,
"29.gate_proj": 1492,
"29.up_proj": 1492,
"29.down_proj": 1492,
"30.q_proj": 1024,
"30.k_proj": 1024,
"30.v_proj": 1024,
"30.o_proj": 1024,
"30.gate_proj": 1492,
"30.up_proj": 1492,
"30.down_proj": 1492,
"31.q_proj": 1024,
"31.k_proj": 1024,
"31.v_proj": 1024,
"31.o_proj": 1024,
"31.gate_proj": 1492,
"31.up_proj": 1492,
"31.down_proj": 1492
},
"rel_err": {
"0.q_proj": 0.056553974747657776,
"0.k_proj": 0.04685795679688454,
"0.v_proj": 0.42462489008903503,
"0.o_proj": 0.41271740198135376,
"0.gate_proj": 0.5058050155639648,
"0.up_proj": 0.5222676992416382,
"0.down_proj": 0.6021890044212341,
"1.q_proj": 0.17382179200649261,
"1.k_proj": 0.15503785014152527,
"1.v_proj": 0.5494375228881836,
"1.o_proj": 0.4885140657424927,
"1.gate_proj": 0.4871184825897217,
"1.up_proj": 0.5492638945579529,
"1.down_proj": 0.6396908164024353,
"2.q_proj": 0.35128143429756165,
"2.k_proj": 0.3372994661331177,
"2.v_proj": 0.5789371728897095,
"2.o_proj": 0.6344954967498779,
"2.gate_proj": 0.5533536672592163,
"2.up_proj": 0.5943166613578796,
"2.down_proj": 0.6493852138519287,
"3.q_proj": 0.45942869782447815,
"3.k_proj": 0.46048206090927124,
"3.v_proj": 0.5989988446235657,
"3.o_proj": 0.6259922385215759,
"3.gate_proj": 0.6128076910972595,
"3.up_proj": 0.637114405632019,
"3.down_proj": 0.6141586899757385,
"4.q_proj": 0.4468260109424591,
"4.k_proj": 0.4523918032646179,
"4.v_proj": 0.5958268046379089,
"4.o_proj": 0.6298879981040955,
"4.gate_proj": 0.6123293042182922,
"4.up_proj": 0.6424803137779236,
"4.down_proj": 0.6184095144271851,
"5.q_proj": 0.4576128125190735,
"5.k_proj": 0.45727330446243286,
"5.v_proj": 0.601065993309021,
"5.o_proj": 0.6338834762573242,
"5.gate_proj": 0.607907235622406,
"5.up_proj": 0.6438080072402954,
"5.down_proj": 0.6109306216239929,
"6.q_proj": 0.4408089220523834,
"6.k_proj": 0.4491904675960541,
"6.v_proj": 0.5842443704605103,
"6.o_proj": 0.6031578779220581,
"6.gate_proj": 0.6021695733070374,
"6.up_proj": 0.6445556879043579,
"6.down_proj": 0.6101555824279785,
"7.q_proj": 0.44747042655944824,
"7.k_proj": 0.45476046204566956,
"7.v_proj": 0.5772215723991394,
"7.o_proj": 0.6210200190544128,
"7.gate_proj": 0.5988063812255859,
"7.up_proj": 0.6436939239501953,
"7.down_proj": 0.6032240390777588,
"8.q_proj": 0.4587515592575073,
"8.k_proj": 0.47478100657463074,
"8.v_proj": 0.561811089515686,
"8.o_proj": 0.6184262037277222,
"8.gate_proj": 0.5957343578338623,
"8.up_proj": 0.6378790140151978,
"8.down_proj": 0.5972688794136047,
"9.q_proj": 0.47541239857673645,
"9.k_proj": 0.4823366403579712,
"9.v_proj": 0.5604491233825684,
"9.o_proj": 0.6180756092071533,
"9.gate_proj": 0.5917431712150574,
"9.up_proj": 0.6320783495903015,
"9.down_proj": 0.595856249332428,
"10.q_proj": 0.4432072639465332,
"10.k_proj": 0.45657336711883545,
"10.v_proj": 0.5517785549163818,
"10.o_proj": 0.5870040655136108,
"10.gate_proj": 0.5866174101829529,
"10.up_proj": 0.6271697282791138,
"10.down_proj": 0.5920713543891907,
"11.q_proj": 0.42947909235954285,
"11.k_proj": 0.4439992606639862,
"11.v_proj": 0.5371034145355225,
"11.o_proj": 0.57439786195755,
"11.gate_proj": 0.5873267650604248,
"11.up_proj": 0.6280611753463745,
"11.down_proj": 0.598749041557312,
"12.q_proj": 0.47172418236732483,
"12.k_proj": 0.46969470381736755,
"12.v_proj": 0.5586328506469727,
"12.o_proj": 0.6156985759735107,
"12.gate_proj": 0.5921249985694885,
"12.up_proj": 0.6282456517219543,
"12.down_proj": 0.5990666151046753,
"13.q_proj": 0.4774617850780487,
"13.k_proj": 0.49627572298049927,
"13.v_proj": 0.5897672176361084,
"13.o_proj": 0.605217695236206,
"13.gate_proj": 0.5953164100646973,
"13.up_proj": 0.6316003799438477,
"13.down_proj": 0.6107012033462524,
"14.q_proj": 0.4548565745353699,
"14.k_proj": 0.45965564250946045,
"14.v_proj": 0.581347644329071,
"14.o_proj": 0.60160893201828,
"14.gate_proj": 0.5952278971672058,
"14.up_proj": 0.6304778456687927,
"14.down_proj": 0.6131455302238464,
"15.q_proj": 0.4785459041595459,
"15.k_proj": 0.48010870814323425,
"15.v_proj": 0.5868794918060303,
"15.o_proj": 0.6144019961357117,
"15.gate_proj": 0.5992725491523743,
"15.up_proj": 0.6341173648834229,
"15.down_proj": 0.6269186735153198,
"16.q_proj": 0.4746119976043701,
"16.k_proj": 0.47853466868400574,
"16.v_proj": 0.6163178086280823,
"16.o_proj": 0.6412844061851501,
"16.gate_proj": 0.5987697839736938,
"16.up_proj": 0.6317082047462463,
"16.down_proj": 0.6406448483467102,
"17.q_proj": 0.5189130902290344,
"17.k_proj": 0.5160570740699768,
"17.v_proj": 0.630569338798523,
"17.o_proj": 0.6520063281059265,
"17.gate_proj": 0.6015799045562744,
"17.up_proj": 0.6326241493225098,
"17.down_proj": 0.6389971971511841,
"18.q_proj": 0.5447173714637756,
"18.k_proj": 0.5495786070823669,
"18.v_proj": 0.6364152431488037,
"18.o_proj": 0.6751506924629211,
"18.gate_proj": 0.6107243299484253,
"18.up_proj": 0.6389372944831848,
"18.down_proj": 0.6388793587684631,
"19.q_proj": 0.5270981788635254,
"19.k_proj": 0.5227535963058472,
"19.v_proj": 0.6196379065513611,
"19.o_proj": 0.661933958530426,
"19.gate_proj": 0.6142390370368958,
"19.up_proj": 0.6393200755119324,
"19.down_proj": 0.6383724808692932,
"20.q_proj": 0.5181286334991455,
"20.k_proj": 0.5196775794029236,
"20.v_proj": 0.6343844532966614,
"20.o_proj": 0.6658429503440857,
"20.gate_proj": 0.6120012998580933,
"20.up_proj": 0.6360871195793152,
"20.down_proj": 0.6502406597137451,
"21.q_proj": 0.5341808795928955,
"21.k_proj": 0.536583662033081,
"21.v_proj": 0.6265413761138916,
"21.o_proj": 0.6791771054267883,
"21.gate_proj": 0.6129226684570312,
"21.up_proj": 0.6356406211853027,
"21.down_proj": 0.6459817290306091,
"22.q_proj": 0.54691481590271,
"22.k_proj": 0.5513964295387268,
"22.v_proj": 0.637810230255127,
"22.o_proj": 0.6779014468193054,
"22.gate_proj": 0.6128952503204346,
"22.up_proj": 0.6347930431365967,
"22.down_proj": 0.6436445116996765,
"23.q_proj": 0.5789421796798706,
"23.k_proj": 0.5783354640007019,
"23.v_proj": 0.6367030739784241,
"23.o_proj": 0.6686539649963379,
"23.gate_proj": 0.6166793704032898,
"23.up_proj": 0.637042224407196,
"23.down_proj": 0.6354413032531738,
"24.q_proj": 0.5257633924484253,
"24.k_proj": 0.5298961400985718,
"24.v_proj": 0.6282287240028381,
"24.o_proj": 0.6736918091773987,
"24.gate_proj": 0.6179255247116089,
"24.up_proj": 0.6380579471588135,
"24.down_proj": 0.6276400089263916,
"25.q_proj": 0.5676007270812988,
"25.k_proj": 0.5755802392959595,
"25.v_proj": 0.637893795967102,
"25.o_proj": 0.6656416058540344,
"25.gate_proj": 0.6214293837547302,
"25.up_proj": 0.6404055953025818,
"25.down_proj": 0.6199561953544617,
"26.q_proj": 0.5362301468849182,
"26.k_proj": 0.5454766154289246,
"26.v_proj": 0.6258969902992249,
"26.o_proj": 0.6879633665084839,
"26.gate_proj": 0.6232463717460632,
"26.up_proj": 0.6424471139907837,
"26.down_proj": 0.6178401708602905,
"27.q_proj": 0.5860579609870911,
"27.k_proj": 0.5940662026405334,
"27.v_proj": 0.647848904132843,
"27.o_proj": 0.6679098606109619,
"27.gate_proj": 0.6263453960418701,
"27.up_proj": 0.6441685557365417,
"27.down_proj": 0.615714430809021,
"28.q_proj": 0.5673431754112244,
"28.k_proj": 0.5781059265136719,
"28.v_proj": 0.643958568572998,
"28.o_proj": 0.6630887389183044,
"28.gate_proj": 0.6261605024337769,
"28.up_proj": 0.6427420377731323,
"28.down_proj": 0.6251272559165955,
"29.q_proj": 0.5318664908409119,
"29.k_proj": 0.5405742526054382,
"29.v_proj": 0.6357550621032715,
"29.o_proj": 0.6679477095603943,
"29.gate_proj": 0.6251234412193298,
"29.up_proj": 0.638621985912323,
"29.down_proj": 0.6331459879875183,
"30.q_proj": 0.5658331513404846,
"30.k_proj": 0.5740678310394287,
"30.v_proj": 0.6490836143493652,
"30.o_proj": 0.6736571192741394,
"30.gate_proj": 0.6186166405677795,
"30.up_proj": 0.6263405084609985,
"30.down_proj": 0.6437183618545532,
"31.q_proj": 0.5029494762420654,
"31.k_proj": 0.5119549632072449,
"31.v_proj": 0.6305423974990845,
"31.o_proj": 0.6436521410942078,
"31.gate_proj": 0.6086755394935608,
"31.up_proj": 0.6120790839195251,
"31.down_proj": 0.6417486667633057
},
"protection": {
"basis": "results/delta_basis/llama2_7b_k256.pt",
"k": 8,
"side": "auto",
"mode": "protect",
"base_model": null,
"basis_meta": {
"chat": "assets/models/Llama-2-7b-chat-hf",
"base": "assets/models/Llama-2-7b-hf",
"k_max": 256,
"oversample": 16,
"niter": 8,
"modules": [
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj"
],
"whitened": false,
"energy_ks": [
1,
4,
16,
64
]
},
"n_protected": 224,
"mean_leak": 5.238032006868057e-06,
"max_leak": 0.00020721335022244602,
"max_protect_err": 7.918087248981465e-06,
"reserved": {
"0.q_proj": 8,
"0.k_proj": 8,
"0.v_proj": 8,
"0.o_proj": 8,
"0.gate_proj": 8,
"0.up_proj": 8,
"0.down_proj": 8,
"1.q_proj": 8,
"1.k_proj": 8,
"1.v_proj": 8,
"1.o_proj": 8,
"1.gate_proj": 8,
"1.up_proj": 8,
"1.down_proj": 8,
"2.q_proj": 8,
"2.k_proj": 8,
"2.v_proj": 8,
"2.o_proj": 8,
"2.gate_proj": 8,
"2.up_proj": 8,
"2.down_proj": 8,
"3.q_proj": 8,
"3.k_proj": 8,
"3.v_proj": 8,
"3.o_proj": 8,
"3.gate_proj": 8,
"3.up_proj": 8,
"3.down_proj": 8,
"4.q_proj": 8,
"4.k_proj": 8,
"4.v_proj": 8,
"4.o_proj": 8,
"4.gate_proj": 8,
"4.up_proj": 8,
"4.down_proj": 8,
"5.q_proj": 8,
"5.k_proj": 8,
"5.v_proj": 8,
"5.o_proj": 8,
"5.gate_proj": 8,
"5.up_proj": 8,
"5.down_proj": 8,
"6.q_proj": 8,
"6.k_proj": 8,
"6.v_proj": 8,
"6.o_proj": 8,
"6.gate_proj": 8,
"6.up_proj": 8,
"6.down_proj": 8,
"7.q_proj": 8,
"7.k_proj": 8,
"7.v_proj": 8,
"7.o_proj": 8,
"7.gate_proj": 8,
"7.up_proj": 8,
"7.down_proj": 8,
"8.q_proj": 8,
"8.k_proj": 8,
"8.v_proj": 8,
"8.o_proj": 8,
"8.gate_proj": 8,
"8.up_proj": 8,
"8.down_proj": 8,
"9.q_proj": 8,
"9.k_proj": 8,
"9.v_proj": 8,
"9.o_proj": 8,
"9.gate_proj": 8,
"9.up_proj": 8,
"9.down_proj": 8,
"10.q_proj": 8,
"10.k_proj": 8,
"10.v_proj": 8,
"10.o_proj": 8,
"10.gate_proj": 8,
"10.up_proj": 8,
"10.down_proj": 8,
"11.q_proj": 8,
"11.k_proj": 8,
"11.v_proj": 8,
"11.o_proj": 8,
"11.gate_proj": 8,
"11.up_proj": 8,
"11.down_proj": 8,
"12.q_proj": 8,
"12.k_proj": 8,
"12.v_proj": 8,
"12.o_proj": 8,
"12.gate_proj": 8,
"12.up_proj": 8,
"12.down_proj": 8,
"13.q_proj": 8,
"13.k_proj": 8,
"13.v_proj": 8,
"13.o_proj": 8,
"13.gate_proj": 8,
"13.up_proj": 8,
"13.down_proj": 8,
"14.q_proj": 8,
"14.k_proj": 8,
"14.v_proj": 8,
"14.o_proj": 8,
"14.gate_proj": 8,
"14.up_proj": 8,
"14.down_proj": 8,
"15.q_proj": 8,
"15.k_proj": 8,
"15.v_proj": 8,
"15.o_proj": 8,
"15.gate_proj": 8,
"15.up_proj": 8,
"15.down_proj": 8,
"16.q_proj": 8,
"16.k_proj": 8,
"16.v_proj": 8,
"16.o_proj": 8,
"16.gate_proj": 8,
"16.up_proj": 8,
"16.down_proj": 8,
"17.q_proj": 8,
"17.k_proj": 8,
"17.v_proj": 8,
"17.o_proj": 8,
"17.gate_proj": 8,
"17.up_proj": 8,
"17.down_proj": 8,
"18.q_proj": 8,
"18.k_proj": 8,
"18.v_proj": 8,
"18.o_proj": 8,
"18.gate_proj": 8,
"18.up_proj": 8,
"18.down_proj": 8,
"19.q_proj": 8,
"19.k_proj": 8,
"19.v_proj": 8,
"19.o_proj": 8,
"19.gate_proj": 8,
"19.up_proj": 8,
"19.down_proj": 8,
"20.q_proj": 8,
"20.k_proj": 8,
"20.v_proj": 8,
"20.o_proj": 8,
"20.gate_proj": 8,
"20.up_proj": 8,
"20.down_proj": 8,
"21.q_proj": 8,
"21.k_proj": 8,
"21.v_proj": 8,
"21.o_proj": 8,
"21.gate_proj": 8,
"21.up_proj": 8,
"21.down_proj": 8,
"22.q_proj": 8,
"22.k_proj": 8,
"22.v_proj": 8,
"22.o_proj": 8,
"22.gate_proj": 8,
"22.up_proj": 8,
"22.down_proj": 8,
"23.q_proj": 8,
"23.k_proj": 8,
"23.v_proj": 8,
"23.o_proj": 8,
"23.gate_proj": 8,
"23.up_proj": 8,
"23.down_proj": 8,
"24.q_proj": 8,
"24.k_proj": 8,
"24.v_proj": 8,
"24.o_proj": 8,
"24.gate_proj": 8,
"24.up_proj": 8,
"24.down_proj": 8,
"25.q_proj": 8,
"25.k_proj": 8,
"25.v_proj": 8,
"25.o_proj": 8,
"25.gate_proj": 8,
"25.up_proj": 8,
"25.down_proj": 8,
"26.q_proj": 8,
"26.k_proj": 8,
"26.v_proj": 8,
"26.o_proj": 8,
"26.gate_proj": 8,
"26.up_proj": 8,
"26.down_proj": 8,
"27.q_proj": 8,
"27.k_proj": 8,
"27.v_proj": 8,
"27.o_proj": 8,
"27.gate_proj": 8,
"27.up_proj": 8,
"27.down_proj": 8,
"28.q_proj": 8,
"28.k_proj": 8,
"28.v_proj": 8,
"28.o_proj": 8,
"28.gate_proj": 8,
"28.up_proj": 8,
"28.down_proj": 8,
"29.q_proj": 8,
"29.k_proj": 8,
"29.v_proj": 8,
"29.o_proj": 8,
"29.gate_proj": 8,
"29.up_proj": 8,
"29.down_proj": 8,
"30.q_proj": 8,
"30.k_proj": 8,
"30.v_proj": 8,
"30.o_proj": 8,
"30.gate_proj": 8,
"30.up_proj": 8,
"30.down_proj": 8,
"31.q_proj": 8,
"31.k_proj": 8,
"31.v_proj": 8,
"31.o_proj": 8,
"31.gate_proj": 8,
"31.up_proj": 8,
"31.down_proj": 8
},
"residual_rank": {
"0.q_proj": 1016,
"0.k_proj": 1016,
"0.v_proj": 1016,
"0.o_proj": 1016,
"0.gate_proj": 1484,
"0.up_proj": 1484,
"0.down_proj": 1484,
"1.q_proj": 1016,
"1.k_proj": 1016,
"1.v_proj": 1016,
"1.o_proj": 1016,
"1.gate_proj": 1484,
"1.up_proj": 1484,
"1.down_proj": 1484,
"2.q_proj": 1016,
"2.k_proj": 1016,
"2.v_proj": 1016,
"2.o_proj": 1016,
"2.gate_proj": 1484,
"2.up_proj": 1484,
"2.down_proj": 1484,
"3.q_proj": 1016,
"3.k_proj": 1016,
"3.v_proj": 1016,
"3.o_proj": 1016,
"3.gate_proj": 1484,
"3.up_proj": 1484,
"3.down_proj": 1484,
"4.q_proj": 1016,
"4.k_proj": 1016,
"4.v_proj": 1016,
"4.o_proj": 1016,
"4.gate_proj": 1484,
"4.up_proj": 1484,
"4.down_proj": 1484,
"5.q_proj": 1016,
"5.k_proj": 1016,
"5.v_proj": 1016,
"5.o_proj": 1016,
"5.gate_proj": 1484,
"5.up_proj": 1484,
"5.down_proj": 1484,
"6.q_proj": 1016,
"6.k_proj": 1016,
"6.v_proj": 1016,
"6.o_proj": 1016,
"6.gate_proj": 1484,
"6.up_proj": 1484,
"6.down_proj": 1484,
"7.q_proj": 1016,
"7.k_proj": 1016,
"7.v_proj": 1016,
"7.o_proj": 1016,
"7.gate_proj": 1484,
"7.up_proj": 1484,
"7.down_proj": 1484,
"8.q_proj": 1016,
"8.k_proj": 1016,
"8.v_proj": 1016,
"8.o_proj": 1016,
"8.gate_proj": 1484,
"8.up_proj": 1484,
"8.down_proj": 1484,
"9.q_proj": 1016,
"9.k_proj": 1016,
"9.v_proj": 1016,
"9.o_proj": 1016,
"9.gate_proj": 1484,
"9.up_proj": 1484,
"9.down_proj": 1484,
"10.q_proj": 1016,
"10.k_proj": 1016,
"10.v_proj": 1016,
"10.o_proj": 1016,
"10.gate_proj": 1484,
"10.up_proj": 1484,
"10.down_proj": 1484,
"11.q_proj": 1016,
"11.k_proj": 1016,
"11.v_proj": 1016,
"11.o_proj": 1016,
"11.gate_proj": 1484,
"11.up_proj": 1484,
"11.down_proj": 1484,
"12.q_proj": 1016,
"12.k_proj": 1016,
"12.v_proj": 1016,
"12.o_proj": 1016,
"12.gate_proj": 1484,
"12.up_proj": 1484,
"12.down_proj": 1484,
"13.q_proj": 1016,
"13.k_proj": 1016,
"13.v_proj": 1016,
"13.o_proj": 1016,
"13.gate_proj": 1484,
"13.up_proj": 1484,
"13.down_proj": 1484,
"14.q_proj": 1016,
"14.k_proj": 1016,
"14.v_proj": 1016,
"14.o_proj": 1016,
"14.gate_proj": 1484,
"14.up_proj": 1484,
"14.down_proj": 1484,
"15.q_proj": 1016,
"15.k_proj": 1016,
"15.v_proj": 1016,
"15.o_proj": 1016,
"15.gate_proj": 1484,
"15.up_proj": 1484,
"15.down_proj": 1484,
"16.q_proj": 1016,
"16.k_proj": 1016,
"16.v_proj": 1016,
"16.o_proj": 1016,
"16.gate_proj": 1484,
"16.up_proj": 1484,
"16.down_proj": 1484,
"17.q_proj": 1016,
"17.k_proj": 1016,
"17.v_proj": 1016,
"17.o_proj": 1016,
"17.gate_proj": 1484,
"17.up_proj": 1484,
"17.down_proj": 1484,
"18.q_proj": 1016,
"18.k_proj": 1016,
"18.v_proj": 1016,
"18.o_proj": 1016,
"18.gate_proj": 1484,
"18.up_proj": 1484,
"18.down_proj": 1484,
"19.q_proj": 1016,
"19.k_proj": 1016,
"19.v_proj": 1016,
"19.o_proj": 1016,
"19.gate_proj": 1484,
"19.up_proj": 1484,
"19.down_proj": 1484,
"20.q_proj": 1016,
"20.k_proj": 1016,
"20.v_proj": 1016,
"20.o_proj": 1016,
"20.gate_proj": 1484,
"20.up_proj": 1484,
"20.down_proj": 1484,
"21.q_proj": 1016,
"21.k_proj": 1016,
"21.v_proj": 1016,
"21.o_proj": 1016,
"21.gate_proj": 1484,
"21.up_proj": 1484,
"21.down_proj": 1484,
"22.q_proj": 1016,
"22.k_proj": 1016,
"22.v_proj": 1016,
"22.o_proj": 1016,
"22.gate_proj": 1484,
"22.up_proj": 1484,
"22.down_proj": 1484,
"23.q_proj": 1016,
"23.k_proj": 1016,
"23.v_proj": 1016,
"23.o_proj": 1016,
"23.gate_proj": 1484,
"23.up_proj": 1484,
"23.down_proj": 1484,
"24.q_proj": 1016,
"24.k_proj": 1016,
"24.v_proj": 1016,
"24.o_proj": 1016,
"24.gate_proj": 1484,
"24.up_proj": 1484,
"24.down_proj": 1484,
"25.q_proj": 1016,
"25.k_proj": 1016,
"25.v_proj": 1016,
"25.o_proj": 1016,
"25.gate_proj": 1484,
"25.up_proj": 1484,
"25.down_proj": 1484,
"26.q_proj": 1016,
"26.k_proj": 1016,
"26.v_proj": 1016,
"26.o_proj": 1016,
"26.gate_proj": 1484,
"26.up_proj": 1484,
"26.down_proj": 1484,
"27.q_proj": 1016,
"27.k_proj": 1016,
"27.v_proj": 1016,
"27.o_proj": 1016,
"27.gate_proj": 1484,
"27.up_proj": 1484,
"27.down_proj": 1484,
"28.q_proj": 1016,
"28.k_proj": 1016,
"28.v_proj": 1016,
"28.o_proj": 1016,
"28.gate_proj": 1484,
"28.up_proj": 1484,
"28.down_proj": 1484,
"29.q_proj": 1016,
"29.k_proj": 1016,
"29.v_proj": 1016,
"29.o_proj": 1016,
"29.gate_proj": 1484,
"29.up_proj": 1484,
"29.down_proj": 1484,
"30.q_proj": 1016,
"30.k_proj": 1016,
"30.v_proj": 1016,
"30.o_proj": 1016,
"30.gate_proj": 1484,
"30.up_proj": 1484,
"30.down_proj": 1484,
"31.q_proj": 1016,
"31.k_proj": 1016,
"31.v_proj": 1016,
"31.o_proj": 1016,
"31.gate_proj": 1484,
"31.up_proj": 1484,
"31.down_proj": 1484
}
},
"ranks_include_reserved": true,
"rank_fix_note": "ranks originally recorded the RESIDUAL rank only; corrected to residual + reserved, which is the true rank of the stored matrix. Weights were always correct -- only this field was wrong."
}