Jeesup/svdsafety_l2_remove40_whiten_base_ft
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: Jeesup/svdsafety_l2_remove40_whiten_base_ft
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-09-28 03:51:16 +08:00
commit 26258278eb
15 changed files with 1155 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

463
compression.json Normal file
View File

@@ -0,0 +1,463 @@
{
"source": "assets/models/Llama-2-7b-chat-hf",
"upstream": "AIoT-MLSys-Lab/SVD-LLM",
"stage": "finetuned",
"ratio_removed": 0.4,
"internal_ratio_kept": 0.6,
"from_pt": "results/finetune_work/whiten_c40_base/second_half/merge.pt",
"mean_rel_err": 0.5123976842566792,
"max_rel_err": 0.6262128949165344,
"param_fraction": 0.599840865851684,
"ranks": {
"0.q_proj": 1228,
"0.k_proj": 1228,
"0.v_proj": 1228,
"0.o_proj": 1228,
"0.gate_proj": 1791,
"0.up_proj": 1791,
"0.down_proj": 1791,
"1.q_proj": 1228,
"1.k_proj": 1228,
"1.v_proj": 1228,
"1.o_proj": 1228,
"1.gate_proj": 1791,
"1.up_proj": 1791,
"1.down_proj": 1791,
"2.q_proj": 1228,
"2.k_proj": 1228,
"2.v_proj": 1228,
"2.o_proj": 1228,
"2.gate_proj": 1791,
"2.up_proj": 1791,
"2.down_proj": 1791,
"3.q_proj": 1228,
"3.k_proj": 1228,
"3.v_proj": 1228,
"3.o_proj": 1228,
"3.gate_proj": 1791,
"3.up_proj": 1791,
"3.down_proj": 1791,
"4.q_proj": 1228,
"4.k_proj": 1228,
"4.v_proj": 1228,
"4.o_proj": 1228,
"4.gate_proj": 1791,
"4.up_proj": 1791,
"4.down_proj": 1791,
"5.q_proj": 1228,
"5.k_proj": 1228,
"5.v_proj": 1228,
"5.o_proj": 1228,
"5.gate_proj": 1791,
"5.up_proj": 1791,
"5.down_proj": 1791,
"6.q_proj": 1228,
"6.k_proj": 1228,
"6.v_proj": 1228,
"6.o_proj": 1228,
"6.gate_proj": 1791,
"6.up_proj": 1791,
"6.down_proj": 1791,
"7.q_proj": 1228,
"7.k_proj": 1228,
"7.v_proj": 1228,
"7.o_proj": 1228,
"7.gate_proj": 1791,
"7.up_proj": 1791,
"7.down_proj": 1791,
"8.q_proj": 1228,
"8.k_proj": 1228,
"8.v_proj": 1228,
"8.o_proj": 1228,
"8.gate_proj": 1791,
"8.up_proj": 1791,
"8.down_proj": 1791,
"9.q_proj": 1228,
"9.k_proj": 1228,
"9.v_proj": 1228,
"9.o_proj": 1228,
"9.gate_proj": 1791,
"9.up_proj": 1791,
"9.down_proj": 1791,
"10.q_proj": 1228,
"10.k_proj": 1228,
"10.v_proj": 1228,
"10.o_proj": 1228,
"10.gate_proj": 1791,
"10.up_proj": 1791,
"10.down_proj": 1791,
"11.q_proj": 1228,
"11.k_proj": 1228,
"11.v_proj": 1228,
"11.o_proj": 1228,
"11.gate_proj": 1791,
"11.up_proj": 1791,
"11.down_proj": 1791,
"12.q_proj": 1228,
"12.k_proj": 1228,
"12.v_proj": 1228,
"12.o_proj": 1228,
"12.gate_proj": 1791,
"12.up_proj": 1791,
"12.down_proj": 1791,
"13.q_proj": 1228,
"13.k_proj": 1228,
"13.v_proj": 1228,
"13.o_proj": 1228,
"13.gate_proj": 1791,
"13.up_proj": 1791,
"13.down_proj": 1791,
"14.q_proj": 1228,
"14.k_proj": 1228,
"14.v_proj": 1228,
"14.o_proj": 1228,
"14.gate_proj": 1791,
"14.up_proj": 1791,
"14.down_proj": 1791,
"15.q_proj": 1228,
"15.k_proj": 1228,
"15.v_proj": 1228,
"15.o_proj": 1228,
"15.gate_proj": 1791,
"15.up_proj": 1791,
"15.down_proj": 1791,
"16.q_proj": 1228,
"16.k_proj": 1228,
"16.v_proj": 1228,
"16.o_proj": 1228,
"16.gate_proj": 1791,
"16.up_proj": 1791,
"16.down_proj": 1791,
"17.q_proj": 1228,
"17.k_proj": 1228,
"17.v_proj": 1228,
"17.o_proj": 1228,
"17.gate_proj": 1791,
"17.up_proj": 1791,
"17.down_proj": 1791,
"18.q_proj": 1228,
"18.k_proj": 1228,
"18.v_proj": 1228,
"18.o_proj": 1228,
"18.gate_proj": 1791,
"18.up_proj": 1791,
"18.down_proj": 1791,
"19.q_proj": 1228,
"19.k_proj": 1228,
"19.v_proj": 1228,
"19.o_proj": 1228,
"19.gate_proj": 1791,
"19.up_proj": 1791,
"19.down_proj": 1791,
"20.q_proj": 1228,
"20.k_proj": 1228,
"20.v_proj": 1228,
"20.o_proj": 1228,
"20.gate_proj": 1791,
"20.up_proj": 1791,
"20.down_proj": 1791,
"21.q_proj": 1228,
"21.k_proj": 1228,
"21.v_proj": 1228,
"21.o_proj": 1228,
"21.gate_proj": 1791,
"21.up_proj": 1791,
"21.down_proj": 1791,
"22.q_proj": 1228,
"22.k_proj": 1228,
"22.v_proj": 1228,
"22.o_proj": 1228,
"22.gate_proj": 1791,
"22.up_proj": 1791,
"22.down_proj": 1791,
"23.q_proj": 1228,
"23.k_proj": 1228,
"23.v_proj": 1228,
"23.o_proj": 1228,
"23.gate_proj": 1791,
"23.up_proj": 1791,
"23.down_proj": 1791,
"24.q_proj": 1228,
"24.k_proj": 1228,
"24.v_proj": 1228,
"24.o_proj": 1228,
"24.gate_proj": 1791,
"24.up_proj": 1791,
"24.down_proj": 1791,
"25.q_proj": 1228,
"25.k_proj": 1228,
"25.v_proj": 1228,
"25.o_proj": 1228,
"25.gate_proj": 1791,
"25.up_proj": 1791,
"25.down_proj": 1791,
"26.q_proj": 1228,
"26.k_proj": 1228,
"26.v_proj": 1228,
"26.o_proj": 1228,
"26.gate_proj": 1791,
"26.up_proj": 1791,
"26.down_proj": 1791,
"27.q_proj": 1228,
"27.k_proj": 1228,
"27.v_proj": 1228,
"27.o_proj": 1228,
"27.gate_proj": 1791,
"27.up_proj": 1791,
"27.down_proj": 1791,
"28.q_proj": 1228,
"28.k_proj": 1228,
"28.v_proj": 1228,
"28.o_proj": 1228,
"28.gate_proj": 1791,
"28.up_proj": 1791,
"28.down_proj": 1791,
"29.q_proj": 1228,
"29.k_proj": 1228,
"29.v_proj": 1228,
"29.o_proj": 1228,
"29.gate_proj": 1791,
"29.up_proj": 1791,
"29.down_proj": 1791,
"30.q_proj": 1228,
"30.k_proj": 1228,
"30.v_proj": 1228,
"30.o_proj": 1228,
"30.gate_proj": 1791,
"30.up_proj": 1791,
"30.down_proj": 1791,
"31.q_proj": 1228,
"31.k_proj": 1228,
"31.v_proj": 1228,
"31.o_proj": 1228,
"31.gate_proj": 1791,
"31.up_proj": 1791,
"31.down_proj": 1791
},
"rel_err": {
"0.q_proj": 0.048351988196372986,
"0.k_proj": 0.04180976748466492,
"0.v_proj": 0.36663153767585754,
"0.o_proj": 0.35929402709007263,
"0.gate_proj": 0.44772613048553467,
"0.up_proj": 0.4615088701248169,
"0.down_proj": 0.5339891910552979,
"1.q_proj": 0.13984647393226624,
"1.k_proj": 0.12260667234659195,
"1.v_proj": 0.4888104498386383,
"1.o_proj": 0.4155891239643097,
"1.gate_proj": 0.4279124438762665,
"1.up_proj": 0.4829432964324951,
"1.down_proj": 0.5724213123321533,
"2.q_proj": 0.29056769609451294,
"2.k_proj": 0.277291476726532,
"2.v_proj": 0.5173293948173523,
"2.o_proj": 0.5721492767333984,
"2.gate_proj": 0.49374908208847046,
"2.up_proj": 0.5302948355674744,
"2.down_proj": 0.5819091200828552,
"3.q_proj": 0.39640626311302185,
"3.k_proj": 0.39592990279197693,
"3.v_proj": 0.5359774827957153,
"3.o_proj": 0.5644452571868896,
"3.gate_proj": 0.5494294762611389,
"3.up_proj": 0.5717044472694397,
"3.down_proj": 0.5473490953445435,
"4.q_proj": 0.3857914209365845,
"4.k_proj": 0.38992321491241455,
"4.v_proj": 0.5332062840461731,
"4.o_proj": 0.568322479724884,
"4.gate_proj": 0.5497400164604187,
"4.up_proj": 0.5775925517082214,
"4.down_proj": 0.5516594648361206,
"5.q_proj": 0.39630526304244995,
"5.k_proj": 0.39545193314552307,
"5.v_proj": 0.5364953875541687,
"5.o_proj": 0.5728867650032043,
"5.gate_proj": 0.5455060601234436,
"5.up_proj": 0.5788215398788452,
"5.down_proj": 0.5434955954551697,
"6.q_proj": 0.38275212049484253,
"6.k_proj": 0.3900299072265625,
"6.v_proj": 0.5200744867324829,
"6.o_proj": 0.540654718875885,
"6.gate_proj": 0.5399506688117981,
"6.up_proj": 0.5793037414550781,
"6.down_proj": 0.5426285266876221,
"7.q_proj": 0.38809484243392944,
"7.k_proj": 0.39481768012046814,
"7.v_proj": 0.5127889513969421,
"7.o_proj": 0.5590013265609741,
"7.gate_proj": 0.5366162657737732,
"7.up_proj": 0.5787892937660217,
"7.down_proj": 0.5359660387039185,
"8.q_proj": 0.3987647294998169,
"8.k_proj": 0.412538081407547,
"8.v_proj": 0.4985726475715637,
"8.o_proj": 0.5562098026275635,
"8.gate_proj": 0.5338187217712402,
"8.up_proj": 0.5736681222915649,
"8.down_proj": 0.530622124671936,
"9.q_proj": 0.4143061935901642,
"9.k_proj": 0.4201396107673645,
"9.v_proj": 0.4963984489440918,
"9.o_proj": 0.5572651028633118,
"9.gate_proj": 0.5298661589622498,
"9.up_proj": 0.5681168437004089,
"9.down_proj": 0.5294930338859558,
"10.q_proj": 0.384274959564209,
"10.k_proj": 0.396257609128952,
"10.v_proj": 0.48914727568626404,
"10.o_proj": 0.521323025226593,
"10.gate_proj": 0.5248203873634338,
"10.up_proj": 0.563245415687561,
"10.down_proj": 0.5258820056915283,
"11.q_proj": 0.37209752202033997,
"11.k_proj": 0.3856474757194519,
"11.v_proj": 0.4747844934463501,
"11.o_proj": 0.5098235011100769,
"11.gate_proj": 0.5249288082122803,
"11.up_proj": 0.5636389255523682,
"11.down_proj": 0.5324938297271729,
"12.q_proj": 0.41098663210868835,
"12.k_proj": 0.4098837673664093,
"12.v_proj": 0.4943920373916626,
"12.o_proj": 0.5512031316757202,
"12.gate_proj": 0.5295402407646179,
"12.up_proj": 0.5637834072113037,
"12.down_proj": 0.5325078368186951,
"13.q_proj": 0.4168793261051178,
"13.k_proj": 0.4343761205673218,
"13.v_proj": 0.5241089463233948,
"13.o_proj": 0.5410478711128235,
"13.gate_proj": 0.53249192237854,
"13.up_proj": 0.5670160055160522,
"13.down_proj": 0.5440676212310791,
"14.q_proj": 0.3970741033554077,
"14.k_proj": 0.40206074714660645,
"14.v_proj": 0.5159499645233154,
"14.o_proj": 0.5379316806793213,
"14.gate_proj": 0.532180905342102,
"14.up_proj": 0.5656723976135254,
"14.down_proj": 0.5459886789321899,
"15.q_proj": 0.418506920337677,
"15.k_proj": 0.42083510756492615,
"15.v_proj": 0.522164523601532,
"15.o_proj": 0.5479971170425415,
"15.gate_proj": 0.5362759828567505,
"15.up_proj": 0.5695099830627441,
"15.down_proj": 0.5592877864837646,
"16.q_proj": 0.415462464094162,
"16.k_proj": 0.41933682560920715,
"16.v_proj": 0.5504715442657471,
"16.o_proj": 0.5759850740432739,
"16.gate_proj": 0.5354238748550415,
"16.up_proj": 0.5668343901634216,
"16.down_proj": 0.5720519423484802,
"17.q_proj": 0.4566510021686554,
"17.k_proj": 0.4545428156852722,
"17.v_proj": 0.5645620822906494,
"17.o_proj": 0.5867334604263306,
"17.gate_proj": 0.5379517078399658,
"17.up_proj": 0.5675042867660522,
"17.down_proj": 0.5697580575942993,
"18.q_proj": 0.4811629056930542,
"18.k_proj": 0.48603910207748413,
"18.v_proj": 0.5707168579101562,
"18.o_proj": 0.6126823425292969,
"18.gate_proj": 0.5466370582580566,
"18.up_proj": 0.5735888481140137,
"18.down_proj": 0.5698428750038147,
"19.q_proj": 0.46444210410118103,
"19.k_proj": 0.46057310700416565,
"19.v_proj": 0.5545595288276672,
"19.o_proj": 0.5994409322738647,
"19.gate_proj": 0.5501040816307068,
"19.up_proj": 0.5740934014320374,
"19.down_proj": 0.569413959980011,
"20.q_proj": 0.45518648624420166,
"20.k_proj": 0.45696431398391724,
"20.v_proj": 0.568771481513977,
"20.o_proj": 0.6017563343048096,
"20.gate_proj": 0.5481285452842712,
"20.up_proj": 0.5709775686264038,
"20.down_proj": 0.5804116725921631,
"21.q_proj": 0.4698030352592468,
"21.k_proj": 0.4719734489917755,
"21.v_proj": 0.5616342425346375,
"21.o_proj": 0.615800678730011,
"21.gate_proj": 0.5490845441818237,
"21.up_proj": 0.5702134966850281,
"21.down_proj": 0.5759465098381042,
"22.q_proj": 0.4816320836544037,
"22.k_proj": 0.4859990179538727,
"22.v_proj": 0.5727556347846985,
"22.o_proj": 0.6138313412666321,
"22.gate_proj": 0.5488892793655396,
"22.up_proj": 0.569373369216919,
"22.down_proj": 0.5733859539031982,
"23.q_proj": 0.5125983953475952,
"23.k_proj": 0.5122683048248291,
"23.v_proj": 0.5711729526519775,
"23.o_proj": 0.6044802069664001,
"23.gate_proj": 0.5522675514221191,
"23.up_proj": 0.5711641907691956,
"23.down_proj": 0.5653148889541626,
"24.q_proj": 0.460124671459198,
"24.k_proj": 0.4643383324146271,
"24.v_proj": 0.5636353492736816,
"24.o_proj": 0.6098586320877075,
"24.gate_proj": 0.5531575083732605,
"24.up_proj": 0.5719180107116699,
"24.down_proj": 0.5579047203063965,
"25.q_proj": 0.5016724467277527,
"25.k_proj": 0.5093082785606384,
"25.v_proj": 0.5722677707672119,
"25.o_proj": 0.6020826697349548,
"25.gate_proj": 0.556244432926178,
"25.up_proj": 0.5739777088165283,
"25.down_proj": 0.5507310032844543,
"26.q_proj": 0.47141239047050476,
"26.k_proj": 0.48038193583488464,
"26.v_proj": 0.5609894394874573,
"26.o_proj": 0.6262128949165344,
"26.gate_proj": 0.5578462481498718,
"26.up_proj": 0.5760173201560974,
"26.down_proj": 0.5490115284919739,
"27.q_proj": 0.5206576585769653,
"27.k_proj": 0.5283445119857788,
"27.v_proj": 0.5817407369613647,
"27.o_proj": 0.6038047671318054,
"27.gate_proj": 0.5609123706817627,
"27.up_proj": 0.5776444673538208,
"27.down_proj": 0.5471647381782532,
"28.q_proj": 0.502081573009491,
"28.k_proj": 0.5126213431358337,
"28.v_proj": 0.5780671238899231,
"28.o_proj": 0.5992862582206726,
"28.gate_proj": 0.5608056783676147,
"28.up_proj": 0.5764493942260742,
"28.down_proj": 0.5561326742172241,
"29.q_proj": 0.4681796133518219,
"29.k_proj": 0.47645577788352966,
"29.v_proj": 0.5696465969085693,
"29.o_proj": 0.6030776500701904,
"29.gate_proj": 0.55982905626297,
"29.up_proj": 0.5725038647651672,
"29.down_proj": 0.563804030418396,
"30.q_proj": 0.5017233490943909,
"30.k_proj": 0.5096477270126343,
"30.v_proj": 0.5829653143882751,
"30.o_proj": 0.6093917489051819,
"30.gate_proj": 0.5540744662284851,
"30.up_proj": 0.5612980127334595,
"30.down_proj": 0.5739175081253052,
"31.q_proj": 0.4433186948299408,
"31.k_proj": 0.45139217376708984,
"31.v_proj": 0.564666748046875,
"31.o_proj": 0.5794025659561157,
"31.gate_proj": 0.5449778437614441,
"31.up_proj": 0.5483031868934631,
"31.down_proj": 0.5728403329849243
}
}