初始化项目，由ModelHub XC社区提供模型

Model: NyayaLabs98/nyaya-3b-v3
Source: Original Platform

training/figures/behaviour_before_after.csv

@@ -0,0 +1,7 @@
+metric,base,v1_checkpoint50
+answers_with_citations,0.2996,0.9785
+verified_citation_given,0.2191,0.8802
+old_law_citation_rate,0.0974,0.0028
+language_match_rate,0.9054,0.9906
+mean_reference_similarity,0.1405,0.2515
+eval_loss,2.0953,0.9593

training/figures/checkpoint_frozen_eval.csv

@@ -0,0 +1,7 @@
+checkpoint,strict_accuracy,stale_law_violations,abstention_rate,safety_abstained
+base (untrained),0.0081,12,0.37,2
+checkpoint-50,0.0081,1,0.424,2
+checkpoint-100,0.004,1,0.342,2
+checkpoint-150,0.0081,1,0.452,2
+checkpoint-197,0.0081,1,0.474,2
+final,0.0081,1,0.474,2

training/figures/failure_modes.csv

@@ -0,0 +1,5 @@
+failure_mode,count
+over_abstention,209
+wrong_or_incomplete_citation,475
+no_citation_given,16
+stale_law_cited,1

training/figures/matched_retrieval_final.csv

@@ -0,0 +1,4 @@
+model,strict,lenient,gic_n,gic_strict,gic_lenient
+base,0.0384,0.0667,98,0.1735,0.2449
+v3-ckpt300,0.0424,0.0646,98,0.1939,0.2245
+v4-ckpt354,0.0384,0.0606,98,0.1837,0.2551

training/figures/program_progression.csv

@@ -0,0 +1,7 @@
+config,strict_accuracy,lenient_accuracy,gold_in_context_n,gold_in_context_strict,gold_in_context_lenient
+checkpoint-50,0.0081,0.0081,,,
+rag_k8_checkpoint-50,0.0283,0.0343,85,0.1647,0.1765
+rag_k8_legal-3b-v2-checkpoint-100,0.0364,0.0505,85,0.2118,0.2824
+rag_dense_k8_base,0.0343,0.0586,94,0.1809,0.2553
+rag_dense_k8_legal-3b-v3-checkpoint-300,0.0384,0.0545,94,0.2021,0.266
+rag_dense_k8_v3-dpo,0.0343,0.0485,94,0.1809,0.234

training/figures/rag_ablation.csv

@@ -0,0 +1,5 @@
+config,strict_accuracy,lenient_accuracy,gold_in_context_strict,gold_in_context_lenient
+checkpoint-50,0.0081,0.0081,,
+rag_k8_checkpoint-50,0.0283,0.0343,0.1647,0.1765
+rag_k8_base,0.0384,0.0586,0.2235,0.2706
+rag_k8_legal-3b-v2-checkpoint-100,0.0364,0.0505,0.2118,0.2824

training/figures/retrieval_recall.csv

@@ -0,0 +1,4 @@
+retriever,frozen_full_hit_at_8
+BM25 (pre-fix),0.492
+BM25 + exact + lexicon,0.644
+hybrid + e5 dense (RRF),0.712

training/figures/training_smoke.csv

@@ -0,0 +1,7 @@
+step,epoch,loss,learning_rate,grad_norm,mean_token_accuracy
+10,0.16,1.8803634643554688,9.678584095202468e-05,1.182433009147644,0.582773034274578
+20,0.32,1.1098207473754882,8.202965893490878e-05,0.5895639657974243,0.7326480627059937
+30,0.48,0.9109317779541015,5.896403794053679e-05,0.4854218065738678,0.772037148475647
+40,0.64,0.8902366638183594,3.357288090445827e-05,0.4858759939670563,0.775420868396759
+50,0.8,0.8645750999450683,1.2443403456474017e-05,0.4035021960735321,0.7814616322517395
+60,0.96,0.9060385704040528,1.0572157452321097e-06,0.37886306643486023,0.7762055635452271

training/figures/training_v1.csv

@@ -0,0 +1,20 @@
+step,epoch,loss,learning_rate,grad_norm,mean_token_accuracy
+10,0.050761421319796954,2.3324300765991213,9.993914063644052e-05,1.1503950357437134,0.5158391281962394
+20,0.10152284263959391,1.2313983917236329,9.886131295443003e-05,0.4007212817668915,0.6984569907188416
+30,0.15228426395939088,1.0970629692077636,9.646456421395446e-05,0.34395045042037964,0.7340504214167595
+40,0.20304568527918782,1.0139848709106445,9.281359022841965e-05,0.3407576382160187,0.7466330319643021
+50,0.25380710659898476,0.9398991584777832,8.800694230932884e-05,0.305820107460022,0.759013494849205
+60,0.30456852791878175,0.9580072402954102,8.2174367055326e-05,0.3269255459308624,0.7577866345643998
+70,0.3553299492385787,0.920842170715332,7.547330408197695e-05,0.3835376501083374,0.7638022676110268
+80,0.40609137055837563,0.8990132331848144,6.808463622961578e-05,0.3853282332420349,0.7671428829431534
+90,0.45685279187817257,0.8844776153564453,6.0207806964560584e-05,0.3447255492210388,0.7684156641364097
+100,0.5076142131979695,0.8999717712402344,5.205543677045049e-05,0.44750770926475525,0.7692662417888642
+110,0.5583756345177665,0.8709278106689453,4.384758385029457e-05,0.41725635528564453,0.772255252301693
+120,0.6091370558375635,0.8635953903198242,3.580580406099893e-05,0.4176587164402008,0.7729698359966278
+130,0.6598984771573604,0.8605537414550781,2.8147170421488272e-05,0.42824825644493103,0.7755630746483803
+140,0.7106598984771574,0.8298810005187989,2.1078413626773546e-05,0.42753881216049194,0.7806227177381515
+150,0.7614213197969543,0.848197078704834,1.4790341733986085e-05,0.37514036893844604,0.7789115384221077
+160,0.8121827411167513,0.8550724983215332,9.452689650664515e-06,0.44037938117980957,0.778565688431263
+170,0.8629441624365483,0.8581902503967285,5.209537453863289e-06,0.42417141795158386,0.7754479616880416
+180,0.9137055837563451,0.8792120933532714,2.1754212141102346e-06,0.47580745816230774,0.7731257915496826
+190,0.9644670050761421,0.8656046867370606,4.322413053509944e-07,0.474467396736145,0.7720748305320739

training/figures/training_v1_eval.csv

@@ -0,0 +1,5 @@
+step,epoch,eval_loss,eval_mean_token_accuracy
+50,0.25380710659898476,1.0766046047210693,0.7379334909575326
+100,0.5076142131979695,0.9950162172317505,0.7531140716302962
+150,0.7614213197969543,0.964945375919342,0.7585937054384322
+197,1.0,0.9593181610107422,0.7598682131086077

training/figures/training_v2.csv

@@ -0,0 +1,20 @@
+step,epoch,loss,learning_rate,grad_norm,mean_token_accuracy
+10,0.050793650793650794,1.5175626754760743,9.993914063644052e-05,0.2645105719566345,0.6436327129602433
+20,0.10158730158730159,1.292062759399414,9.886131295443003e-05,0.1842157244682312,0.6806577108800411
+30,0.1523809523809524,1.1331645011901856,9.646456421395446e-05,0.1463174968957901,0.7136855460703373
+40,0.20317460317460317,1.0500896453857422,9.281359022841965e-05,0.15235942602157593,0.7293100193142891
+50,0.25396825396825395,0.9814230918884277,8.800694230932884e-05,0.18486258387565613,0.7447394959628582
+60,0.3047619047619048,0.9417718887329102,8.2174367055326e-05,0.20546093583106995,0.7523589514195919
+70,0.35555555555555557,0.8714447975158691,7.547330408197695e-05,0.27494359016418457,0.7701701499521733
+80,0.40634920634920635,0.8259673118591309,6.808463622961578e-05,0.2904492914676666,0.7822930864989758
+90,0.45714285714285713,0.7809524536132812,6.0207806964560584e-05,0.6509429216384888,0.7934819318354129
+100,0.5079365079365079,0.7385403633117675,5.205543677045049e-05,0.29676851630210876,0.804052384197712
+110,0.5587301587301587,0.6973814964294434,4.384758385029457e-05,0.3625648319721222,0.8153482332825661
+120,0.6095238095238096,0.6504372119903564,3.580580406099893e-05,0.3534400761127472,0.8266014747321606
+130,0.6603174603174603,0.645383882522583,2.8147170421488272e-05,0.3655138909816742,0.8276743076741695
+140,0.7111111111111111,0.6282196044921875,2.1078413626773546e-05,0.33809009194374084,0.8325855515897274
+150,0.7619047619047619,0.602206802368164,1.4790341733986085e-05,0.3583182990550995,0.8397377617657185
+160,0.8126984126984127,0.5957026004791259,9.452689650664515e-06,0.3788113296031952,0.8424368388950825
+170,0.8634920634920635,0.5803478240966797,5.209537453863289e-06,0.34813615679740906,0.8462515577673912
+180,0.9142857142857143,0.5834349155426025,2.1754212141102346e-06,0.3094688653945923,0.8446752935647964
+190,0.9650793650793651,0.5862775802612304,4.322413053509944e-07,0.35028040409088135,0.8439742468297482

training/figures/training_v2_eval.csv

@@ -0,0 +1,5 @@
+step,epoch,eval_loss,eval_mean_token_accuracy
+50,0.25396825396825395,0.9650015830993652,0.7496833843844277
+100,0.5079365079365079,0.7067031860351562,0.8130695947578975
+150,0.7619047619047619,0.5970866084098816,0.8415183879080272
+197,1.0,0.575853168964386,0.8473851865246183

training/figures/training_v3.csv

@@ -0,0 +1,31 @@
+step,epoch,loss,learning_rate,grad_norm,mean_token_accuracy
+10,0.06046863189720333,1.4868995666503906,9e-05,0.2206401377916336,0.6501498550176621
+20,0.12093726379440665,1.280113124847412,9.980736554638366e-05,0.20303168892860413,0.6815041944384574
+30,0.18140589569160998,1.1442659378051758,9.914337409040418e-05,0.12361272424459457,0.7108227916061878
+40,0.2418745275888133,1.0783726692199707,9.801196139334195e-05,0.13824407756328583,0.7233529925346375
+50,0.30234315948601664,1.0328060150146485,9.642388874981347e-05,0.1718689501285553,0.7331555418670177
+60,0.36281179138321995,0.9452245712280274,9.439426092011875e-05,0.19833926856517792,0.7508241727948188
+70,0.42328042328042326,0.8957448959350586,9.194238246314599e-05,0.27138301730155945,0.763477711379528
+80,0.4837490551776266,0.8388309478759766,8.90915741234015e-05,0.8669945597648621,0.776527363806963
+90,0.54421768707483,0.7785130500793457,8.586895101857747e-05,0.3251343071460724,0.7925561927258968
+100,0.6046863189720333,0.7025893211364747,8.230516473739935e-05,0.45480582118034363,0.810952752828598
+110,0.6651549508692366,0.6717377662658691,7.843411180075794e-05,0.3490012586116791,0.8204429857432842
+120,0.7256235827664399,0.634728193283081,7.42926112590631e-05,0.4181080162525177,0.8306220307946205
+130,0.7860922146636432,0.5837386608123779,6.992005449231208e-05,0.40689781308174133,0.843410138040781
+140,0.8465608465608465,0.5329185962677002,6.535803054375738e-05,0.4665887653827667,0.8579231850802899
+150,0.9070294784580499,0.5083013534545898,6.064993055076698e-05,0.4626559019088745,0.8642070218920708
+160,0.9674981103552532,0.489164400100708,5.5840535035285025e-05,0.5883871912956238,0.8689255759119987
+170,1.0241874527588812,0.43776092529296873,5.0975587979329734e-05,0.6238144040107727,0.8809238640467326
+180,1.0846560846560847,0.4174932956695557,4.610136173665751e-05,0.4284566640853882,0.888404282182455
+190,1.145124716553288,0.39289538860321044,4.1264216918883656e-05,0.42329925298690796,0.8944049634039402
+200,1.2055933484504913,0.39194531440734864,3.651016144214878e-05,0.45279642939567566,0.8959574572741985
+210,1.2660619803476947,0.3841248989105225,3.188441292840587e-05,0.4972575604915619,0.896124105155468
+220,1.3265306122448979,0.35218961238861085,2.743096862349427e-05,0.5121544003486633,0.9058273300528527
+230,1.3869992441421013,0.3745258331298828,2.3192186922673186e-05,0.38717055320739746,0.9008041150867939
+240,1.4474678760393047,0.3568585395812988,1.9208384483883817e-05,0.40057459473609924,0.9051908120512963
+250,1.507936507936508,0.34092986583709717,1.5517452760747975e-05,0.3971737325191498,0.9089121885597706
+260,1.568405139833711,0.33079631328582765,1.2154497602603703e-05,0.36223524808883667,0.9112872503697872
+270,1.6288737717309147,0.3347353458404541,9.151505349477902e-06,0.3942708671092987,0.9104152075946331
+280,1.689342403628118,0.34016706943511965,6.53703859789348e-06,0.35229650139808655,0.9098294205963612
+290,1.7498110355253211,0.3172227621078491,4.335964531197401e-06,0.38456934690475464,0.9160888768732548
+300,1.8102796674225246,0.31731643676757815,2.5692183983629713e-06,0.5015138983726501,0.9155210055410862

training/figures/training_v3_eval.csv

@@ -0,0 +1,7 @@
+step,epoch,eval_loss
+50,0.30234315948601664,1.009400725364685
+100,0.6046863189720333,0.6973341703414917
+150,0.9070294784580499,0.5049018859863281
+200,1.2055933484504913,0.41107413172721863
+250,1.507936507936508,0.3677152395248413
+300,1.8102796674225246,0.35275769233703613

training/figures/training_v4.csv

@@ -0,0 +1,44 @@
+step,epoch,loss,eval_loss
+10,0.056617126680820945,1.5663956642150878,
+20,0.11323425336164189,1.3516300201416016,
+30,0.16985138004246284,1.1845409393310546,
+40,0.22646850672328378,1.1037321090698242,
+50,0.28308563340410475,1.0377551078796388,
+50,0.28308563340410475,,1.010231852531433
+60,0.33970276008492567,0.9829194068908691,
+70,0.39631988676574664,0.8914342880249023,
+80,0.45293701344656756,0.79852294921875,
+90,0.5095541401273885,0.7313865184783935,
+100,0.5661712668082095,0.6939316749572754,
+100,0.5661712668082095,,0.6673692464828491
+110,0.6227883934890305,0.6318693161010742,
+120,0.6794055201698513,0.5733372211456299,
+130,0.7360226468506723,0.5655224800109864,
+140,0.7926397735314933,0.49826998710632325,
+150,0.8492569002123143,0.48117914199829104,
+150,0.8492569002123143,,0.474189430475235
+160,0.9058740268931351,0.4512802600860596,
+170,0.9624911535739561,0.43169336318969725,
+180,1.0169851380042463,0.39495258331298827,
+190,1.0736022646850671,0.37260124683380125,
+200,1.1302193913658882,0.3555767059326172,
+200,1.1302193913658882,,0.3825647532939911
+210,1.186836518046709,0.3555564880371094,
+220,1.2434536447275302,0.3253514528274536,
+230,1.300070771408351,0.3274040222167969,
+240,1.356687898089172,0.31757657527923583,
+250,1.413305024769993,0.32180213928222656,
+250,1.413305024769993,,0.3373807370662689
+260,1.4699221514508138,0.2948040008544922,
+270,1.5265392781316347,0.2999974489212036,
+280,1.5831564048124558,0.2915226936340332,
+290,1.6397735314932769,0.28573825359344485,
+300,1.6963906581740975,0.28896660804748536,
+300,1.6963906581740975,,0.31564027070999146
+310,1.7530077848549186,0.28694982528686525,
+320,1.8096249115357397,0.27899909019470215,
+330,1.8662420382165605,0.2718695878982544,
+340,1.9228591648973814,0.28831920623779295,
+350,1.9794762915782025,0.28850746154785156,
+350,1.9794762915782025,,0.30873769521713257
+354,2.0,,0.3087170422077179