NyayaLabs98/nyaya-3b-v3
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: NyayaLabs98/nyaya-3b-v3
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-27 22:08:26 +08:00
commit ff5f5ef338
37 changed files with 2009 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
training/checkpoint_evals.json Normal file
View File

@@ -0,0 +1,151 @@
{
"run_dir": "outputs/legal-3b-v1",
"model": "Qwen/Qwen2.5-3B-Instruct",
"frozen_eval_questions": 500,
"checkpoints": {
"checkpoint-50": {
"strict_accuracy": 0.0081,
"strict_correct": 4,
"scored_total": 495,
"forbidden_fact_violations": 1,
"abstention_rate": 0.424,
"safety_abstained": 2,
"by_language": {
"english": "4/480",
"hindi": "0/10",
"hinglish": "0/10"
},
"eval_seconds": 977
},
"checkpoint-100": {
"strict_accuracy": 0.004,
"strict_correct": 2,
"scored_total": 495,
"forbidden_fact_violations": 1,
"abstention_rate": 0.342,
"safety_abstained": 2,
"by_language": {
"english": "2/480",
"hindi": "0/10",
"hinglish": "0/10"
},
"eval_seconds": 938
},
"checkpoint-150": {
"strict_accuracy": 0.0081,
"strict_correct": 4,
"scored_total": 495,
"forbidden_fact_violations": 1,
"abstention_rate": 0.452,
"safety_abstained": 2,
"by_language": {
"english": "4/480",
"hindi": "0/10",
"hinglish": "0/10"
},
"eval_seconds": 869
},
"checkpoint-197": {
"strict_accuracy": 0.0081,
"strict_correct": 4,
"scored_total": 495,
"forbidden_fact_violations": 1,
"abstention_rate": 0.474,
"safety_abstained": 2,
"by_language": {
"english": "4/480",
"hindi": "0/10",
"hinglish": "0/10"
},
"eval_seconds": 869
},
"final": {
"strict_accuracy": 0.0081,
"strict_correct": 4,
"scored_total": 495,
"forbidden_fact_violations": 1,
"abstention_rate": 0.474,
"safety_abstained": 2,
"by_language": {
"english": "4/480",
"hindi": "0/10",
"hinglish": "0/10"
},
"eval_seconds": 868
}
},
"best": "checkpoint-50",
"best_dataset_eval": {
"total": 1068,
"citation_pass_rate": 0.9017,
"answers_with_citations": 0.9785,
"old_law_citation_rate": 0.0028,
"language_match_rate": 0.9906,
"mean_reference_similarity": 0.2515,
"by_language": {
"english": {
"total": 785,
"citation_ok": 707,
"language_ok": 784,
"old_law_cited": 3
},
"hinglish": {
"total": 168,
"citation_ok": 151,
"language_ok": 167,
"old_law_cited": 0
},
"hindi": {
"total": 115,
"citation_ok": 105,
"language_ok": 107,
"old_law_cited": 0
}
},
"by_task_type": {
"grounded_qa": {
"total": 478,
"citation_ok": 429,
"language_ok": 477,
"old_law_cited": 0
},
"terminology": {
"total": 28,
"citation_ok": 27,
"language_ok": 28,
"old_law_cited": 0
},
"hinglish_qa": {
"total": 168,
"citation_ok": 151,
"language_ok": 167,
"old_law_cited": 0
},
"law_mapping": {
"total": 24,
"citation_ok": 20,
"language_ok": 24,
"old_law_cited": 3
},
"hindi_qa": {
"total": 115,
"citation_ok": 105,
"language_ok": 107,
"old_law_cited": 0
},
"procedural": {
"total": 231,
"citation_ok": 211,
"language_ok": 231,
"old_law_cited": 0
},
"safety_abstention": {
"total": 24,
"citation_ok": 20,
"language_ok": 24,
"old_law_cited": 0
}
}
},
"evaluated_at": "2026-07-15 08:04:21"
}

300
training/dpo_train_report.json Normal file
View File

@@ -0,0 +1,300 @@
{
"pairs": 124,
"sft_adapter": "outputs/legal-3b-v3/checkpoint-300",
"beta": 0.1,
"epochs": 3,
"optimizer_steps": 48,
"final_loss": 0.5268,
"final_margin": 0.366,
"history": [
{
"step": 1,
"epoch": 0,
"loss": 0.6931,
"margin": 0.0
},
{
"step": 2,
"epoch": 0,
"loss": 0.7271,
"margin": -0.0668
},
{
"step": 3,
"epoch": 0,
"loss": 0.6067,
"margin": 0.1811
},
{
"step": 4,
"epoch": 0,
"loss": 0.6631,
"margin": 0.0611
},
{
"step": 5,
"epoch": 0,
"loss": 0.7499,
"margin": -0.1104
},
{
"step": 6,
"epoch": 0,
"loss": 0.6343,
"margin": 0.1214
},
{
"step": 7,
"epoch": 0,
"loss": 0.7168,
"margin": -0.0468
},
{
"step": 8,
"epoch": 0,
"loss": 0.6645,
"margin": 0.0581
},
{
"step": 9,
"epoch": 0,
"loss": 0.6587,
"margin": 0.0701
},
{
"step": 10,
"epoch": 0,
"loss": 0.6732,
"margin": 0.0402
},
{
"step": 11,
"epoch": 0,
"loss": 0.589,
"margin": 0.2205
},
{
"step": 12,
"epoch": 0,
"loss": 0.6851,
"margin": 0.0162
},
{
"step": 13,
"epoch": 0,
"loss": 0.6826,
"margin": 0.0212
},
{
"step": 14,
"epoch": 0,
"loss": 0.5937,
"margin": 0.2099
},
{
"step": 15,
"epoch": 0,
"loss": 0.759,
"margin": -0.1277
},
{
"step": 16,
"epoch": 0,
"loss": 0.7671,
"margin": -0.1427
},
{
"step": 17,
"epoch": 1,
"loss": 0.5843,
"margin": 0.231
},
{
"step": 18,
"epoch": 1,
"loss": 0.7993,
"margin": -0.2021
},
{
"step": 19,
"epoch": 1,
"loss": 0.6293,
"margin": 0.132
},
{
"step": 20,
"epoch": 1,
"loss": 0.7464,
"margin": -0.1038
},
{
"step": 21,
"epoch": 1,
"loss": 0.6757,
"margin": 0.0353
},
{
"step": 22,
"epoch": 1,
"loss": 0.713,
"margin": -0.0393
},
{
"step": 23,
"epoch": 1,
"loss": 0.5206,
"margin": 0.3812
},
{
"step": 24,
"epoch": 1,
"loss": 0.7103,
"margin": -0.034
},
{
"step": 25,
"epoch": 1,
"loss": 0.6525,
"margin": 0.083
},
{
"step": 26,
"epoch": 1,
"loss": 0.63,
"margin": 0.1305
},
{
"step": 27,
"epoch": 1,
"loss": 0.5864,
"margin": 0.2263
},
{
"step": 28,
"epoch": 1,
"loss": 0.4905,
"margin": 0.457
},
{
"step": 29,
"epoch": 1,
"loss": 0.6076,
"margin": 0.1791
},
{
"step": 30,
"epoch": 1,
"loss": 0.6098,
"margin": 0.1743
},
{
"step": 31,
"epoch": 1,
"loss": 0.4762,
"margin": 0.4945
},
{
"step": 32,
"epoch": 1,
"loss": 0.5189,
"margin": 0.3854
},
{
"step": 33,
"epoch": 2,
"loss": 0.5235,
"margin": 0.374
},
{
"step": 34,
"epoch": 2,
"loss": 0.3293,
"margin": 0.9416
},
{
"step": 35,
"epoch": 2,
"loss": 0.4656,
"margin": 0.5226
},
{
"step": 36,
"epoch": 2,
"loss": 0.6588,
"margin": 0.0699
},
{
"step": 37,
"epoch": 2,
"loss": 0.4392,
"margin": 0.5951
},
{
"step": 38,
"epoch": 2,
"loss": 0.5389,
"margin": 0.3368
},
{
"step": 39,
"epoch": 2,
"loss": 0.7196,
"margin": -0.0523
},
{
"step": 40,
"epoch": 2,
"loss": 0.4093,
"margin": 0.6817
},
{
"step": 41,
"epoch": 2,
"loss": 0.5289,
"margin": 0.3609
},
{
"step": 42,
"epoch": 2,
"loss": 0.4894,
"margin": 0.4599
},
{
"step": 43,
"epoch": 2,
"loss": 0.6129,
"margin": 0.1675
},
{
"step": 44,
"epoch": 2,
"loss": 0.5136,
"margin": 0.3986
},
{
"step": 45,
"epoch": 2,
"loss": 0.3373,
"margin": 0.9133
},
{
"step": 46,
"epoch": 2,
"loss": 0.4089,
"margin": 0.683
},
{
"step": 47,
"epoch": 2,
"loss": 0.2918,
"margin": 1.0822
},
{
"step": 48,
"epoch": 2,
"loss": 0.5268,
"margin": 0.366
}
],
"seconds": 760
}

7
training/figures/behaviour_before_after.csv Normal file
View File

@@ -0,0 +1,7 @@
metric,base,v1_checkpoint50
answers_with_citations,0.2996,0.9785
verified_citation_given,0.2191,0.8802
old_law_citation_rate,0.0974,0.0028
language_match_rate,0.9054,0.9906
mean_reference_similarity,0.1405,0.2515
eval_loss,2.0953,0.9593
1 metric base v1_checkpoint50
2 answers_with_citations 0.2996 0.9785
3 verified_citation_given 0.2191 0.8802
4 old_law_citation_rate 0.0974 0.0028
5 language_match_rate 0.9054 0.9906
6 mean_reference_similarity 0.1405 0.2515
7 eval_loss 2.0953 0.9593

BIN
training/figures/behaviour_before_after.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 50 KiB

7
training/figures/checkpoint_frozen_eval.csv Normal file
View File

@@ -0,0 +1,7 @@
checkpoint,strict_accuracy,stale_law_violations,abstention_rate,safety_abstained
base (untrained),0.0081,12,0.37,2
checkpoint-50,0.0081,1,0.424,2
checkpoint-100,0.004,1,0.342,2
checkpoint-150,0.0081,1,0.452,2
checkpoint-197,0.0081,1,0.474,2
final,0.0081,1,0.474,2
1 checkpoint strict_accuracy stale_law_violations abstention_rate safety_abstained
2 base (untrained) 0.0081 12 0.37 2
3 checkpoint-50 0.0081 1 0.424 2
4 checkpoint-100 0.004 1 0.342 2
5 checkpoint-150 0.0081 1 0.452 2
6 checkpoint-197 0.0081 1 0.474 2
7 final 0.0081 1 0.474 2

BIN
training/figures/checkpoint_frozen_eval.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 50 KiB

5
training/figures/failure_modes.csv Normal file
View File

@@ -0,0 +1,5 @@
failure_mode,count
over_abstention,209
wrong_or_incomplete_citation,475
no_citation_given,16
stale_law_cited,1
1 failure_mode count
2 over_abstention 209
3 wrong_or_incomplete_citation 475
4 no_citation_given 16
5 stale_law_cited 1

BIN
training/figures/failure_modes.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 32 KiB

4
training/figures/matched_retrieval_final.csv Normal file
View File

@@ -0,0 +1,4 @@
model,strict,lenient,gic_n,gic_strict,gic_lenient
base,0.0384,0.0667,98,0.1735,0.2449
v3-ckpt300,0.0424,0.0646,98,0.1939,0.2245
v4-ckpt354,0.0384,0.0606,98,0.1837,0.2551
1 model strict lenient gic_n gic_strict gic_lenient
2 base 0.0384 0.0667 98 0.1735 0.2449
3 v3-ckpt300 0.0424 0.0646 98 0.1939 0.2245
4 v4-ckpt354 0.0384 0.0606 98 0.1837 0.2551

7
training/figures/program_progression.csv Normal file
View File

@@ -0,0 +1,7 @@
config,strict_accuracy,lenient_accuracy,gold_in_context_n,gold_in_context_strict,gold_in_context_lenient
checkpoint-50,0.0081,0.0081,,,
rag_k8_checkpoint-50,0.0283,0.0343,85,0.1647,0.1765
rag_k8_legal-3b-v2-checkpoint-100,0.0364,0.0505,85,0.2118,0.2824
rag_dense_k8_base,0.0343,0.0586,94,0.1809,0.2553
rag_dense_k8_legal-3b-v3-checkpoint-300,0.0384,0.0545,94,0.2021,0.266
rag_dense_k8_v3-dpo,0.0343,0.0485,94,0.1809,0.234
1 config strict_accuracy lenient_accuracy gold_in_context_n gold_in_context_strict gold_in_context_lenient
2 checkpoint-50 0.0081 0.0081
3 rag_k8_checkpoint-50 0.0283 0.0343 85 0.1647 0.1765
4 rag_k8_legal-3b-v2-checkpoint-100 0.0364 0.0505 85 0.2118 0.2824
5 rag_dense_k8_base 0.0343 0.0586 94 0.1809 0.2553
6 rag_dense_k8_legal-3b-v3-checkpoint-300 0.0384 0.0545 94 0.2021 0.266
7 rag_dense_k8_v3-dpo 0.0343 0.0485 94 0.1809 0.234

BIN
training/figures/program_progression.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 71 KiB

5
training/figures/rag_ablation.csv Normal file
View File

@@ -0,0 +1,5 @@
config,strict_accuracy,lenient_accuracy,gold_in_context_strict,gold_in_context_lenient
checkpoint-50,0.0081,0.0081,,
rag_k8_checkpoint-50,0.0283,0.0343,0.1647,0.1765
rag_k8_base,0.0384,0.0586,0.2235,0.2706
rag_k8_legal-3b-v2-checkpoint-100,0.0364,0.0505,0.2118,0.2824
1 config strict_accuracy lenient_accuracy gold_in_context_strict gold_in_context_lenient
2 checkpoint-50 0.0081 0.0081
3 rag_k8_checkpoint-50 0.0283 0.0343 0.1647 0.1765
4 rag_k8_base 0.0384 0.0586 0.2235 0.2706
5 rag_k8_legal-3b-v2-checkpoint-100 0.0364 0.0505 0.2118 0.2824

BIN
training/figures/rag_ablation.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 57 KiB

4
training/figures/retrieval_recall.csv Normal file
View File

@@ -0,0 +1,4 @@
retriever,frozen_full_hit_at_8
BM25 (pre-fix),0.492
BM25 + exact + lexicon,0.644
hybrid + e5 dense (RRF),0.712
1 retriever frozen_full_hit_at_8
2 BM25 (pre-fix) 0.492
3 BM25 + exact + lexicon 0.644
4 hybrid + e5 dense (RRF) 0.712

BIN
training/figures/smoke_vs_v1_loss.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 49 KiB

7
training/figures/training_smoke.csv Normal file
View File

@@ -0,0 +1,7 @@
step,epoch,loss,learning_rate,grad_norm,mean_token_accuracy
10,0.16,1.8803634643554688,9.678584095202468e-05,1.182433009147644,0.582773034274578
20,0.32,1.1098207473754882,8.202965893490878e-05,0.5895639657974243,0.7326480627059937
30,0.48,0.9109317779541015,5.896403794053679e-05,0.4854218065738678,0.772037148475647
40,0.64,0.8902366638183594,3.357288090445827e-05,0.4858759939670563,0.775420868396759
50,0.8,0.8645750999450683,1.2443403456474017e-05,0.4035021960735321,0.7814616322517395
60,0.96,0.9060385704040528,1.0572157452321097e-06,0.37886306643486023,0.7762055635452271
1 step epoch loss learning_rate grad_norm mean_token_accuracy
2 10 0.16 1.8803634643554688 9.678584095202468e-05 1.182433009147644 0.582773034274578
3 20 0.32 1.1098207473754882 8.202965893490878e-05 0.5895639657974243 0.7326480627059937
4 30 0.48 0.9109317779541015 5.896403794053679e-05 0.4854218065738678 0.772037148475647
5 40 0.64 0.8902366638183594 3.357288090445827e-05 0.4858759939670563 0.775420868396759
6 50 0.8 0.8645750999450683 1.2443403456474017e-05 0.4035021960735321 0.7814616322517395
7 60 0.96 0.9060385704040528 1.0572157452321097e-06 0.37886306643486023 0.7762055635452271

20
training/figures/training_v1.csv Normal file
View File

@@ -0,0 +1,20 @@
step,epoch,loss,learning_rate,grad_norm,mean_token_accuracy
10,0.050761421319796954,2.3324300765991213,9.993914063644052e-05,1.1503950357437134,0.5158391281962394
20,0.10152284263959391,1.2313983917236329,9.886131295443003e-05,0.4007212817668915,0.6984569907188416
30,0.15228426395939088,1.0970629692077636,9.646456421395446e-05,0.34395045042037964,0.7340504214167595
40,0.20304568527918782,1.0139848709106445,9.281359022841965e-05,0.3407576382160187,0.7466330319643021
50,0.25380710659898476,0.9398991584777832,8.800694230932884e-05,0.305820107460022,0.759013494849205
60,0.30456852791878175,0.9580072402954102,8.2174367055326e-05,0.3269255459308624,0.7577866345643998
70,0.3553299492385787,0.920842170715332,7.547330408197695e-05,0.3835376501083374,0.7638022676110268
80,0.40609137055837563,0.8990132331848144,6.808463622961578e-05,0.3853282332420349,0.7671428829431534
90,0.45685279187817257,0.8844776153564453,6.0207806964560584e-05,0.3447255492210388,0.7684156641364097
100,0.5076142131979695,0.8999717712402344,5.205543677045049e-05,0.44750770926475525,0.7692662417888642
110,0.5583756345177665,0.8709278106689453,4.384758385029457e-05,0.41725635528564453,0.772255252301693
120,0.6091370558375635,0.8635953903198242,3.580580406099893e-05,0.4176587164402008,0.7729698359966278
130,0.6598984771573604,0.8605537414550781,2.8147170421488272e-05,0.42824825644493103,0.7755630746483803
140,0.7106598984771574,0.8298810005187989,2.1078413626773546e-05,0.42753881216049194,0.7806227177381515
150,0.7614213197969543,0.848197078704834,1.4790341733986085e-05,0.37514036893844604,0.7789115384221077
160,0.8121827411167513,0.8550724983215332,9.452689650664515e-06,0.44037938117980957,0.778565688431263
170,0.8629441624365483,0.8581902503967285,5.209537453863289e-06,0.42417141795158386,0.7754479616880416
180,0.9137055837563451,0.8792120933532714,2.1754212141102346e-06,0.47580745816230774,0.7731257915496826
190,0.9644670050761421,0.8656046867370606,4.322413053509944e-07,0.474467396736145,0.7720748305320739
1 step epoch loss learning_rate grad_norm mean_token_accuracy
2 10 0.050761421319796954 2.3324300765991213 9.993914063644052e-05 1.1503950357437134 0.5158391281962394
3 20 0.10152284263959391 1.2313983917236329 9.886131295443003e-05 0.4007212817668915 0.6984569907188416
4 30 0.15228426395939088 1.0970629692077636 9.646456421395446e-05 0.34395045042037964 0.7340504214167595
5 40 0.20304568527918782 1.0139848709106445 9.281359022841965e-05 0.3407576382160187 0.7466330319643021
6 50 0.25380710659898476 0.9398991584777832 8.800694230932884e-05 0.305820107460022 0.759013494849205
7 60 0.30456852791878175 0.9580072402954102 8.2174367055326e-05 0.3269255459308624 0.7577866345643998
8 70 0.3553299492385787 0.920842170715332 7.547330408197695e-05 0.3835376501083374 0.7638022676110268
9 80 0.40609137055837563 0.8990132331848144 6.808463622961578e-05 0.3853282332420349 0.7671428829431534
10 90 0.45685279187817257 0.8844776153564453 6.0207806964560584e-05 0.3447255492210388 0.7684156641364097
11 100 0.5076142131979695 0.8999717712402344 5.205543677045049e-05 0.44750770926475525 0.7692662417888642
12 110 0.5583756345177665 0.8709278106689453 4.384758385029457e-05 0.41725635528564453 0.772255252301693
13 120 0.6091370558375635 0.8635953903198242 3.580580406099893e-05 0.4176587164402008 0.7729698359966278
14 130 0.6598984771573604 0.8605537414550781 2.8147170421488272e-05 0.42824825644493103 0.7755630746483803
15 140 0.7106598984771574 0.8298810005187989 2.1078413626773546e-05 0.42753881216049194 0.7806227177381515
16 150 0.7614213197969543 0.848197078704834 1.4790341733986085e-05 0.37514036893844604 0.7789115384221077
17 160 0.8121827411167513 0.8550724983215332 9.452689650664515e-06 0.44037938117980957 0.778565688431263
18 170 0.8629441624365483 0.8581902503967285 5.209537453863289e-06 0.42417141795158386 0.7754479616880416
19 180 0.9137055837563451 0.8792120933532714 2.1754212141102346e-06 0.47580745816230774 0.7731257915496826
20 190 0.9644670050761421 0.8656046867370606 4.322413053509944e-07 0.474467396736145 0.7720748305320739

5
training/figures/training_v1_eval.csv Normal file
View File

@@ -0,0 +1,5 @@
step,epoch,eval_loss,eval_mean_token_accuracy
50,0.25380710659898476,1.0766046047210693,0.7379334909575326
100,0.5076142131979695,0.9950162172317505,0.7531140716302962
150,0.7614213197969543,0.964945375919342,0.7585937054384322
197,1.0,0.9593181610107422,0.7598682131086077
1 step epoch eval_loss eval_mean_token_accuracy
2 50 0.25380710659898476 1.0766046047210693 0.7379334909575326
3 100 0.5076142131979695 0.9950162172317505 0.7531140716302962
4 150 0.7614213197969543 0.964945375919342 0.7585937054384322
5 197 1.0 0.9593181610107422 0.7598682131086077

20
training/figures/training_v2.csv Normal file
View File

@@ -0,0 +1,20 @@
step,epoch,loss,learning_rate,grad_norm,mean_token_accuracy
10,0.050793650793650794,1.5175626754760743,9.993914063644052e-05,0.2645105719566345,0.6436327129602433
20,0.10158730158730159,1.292062759399414,9.886131295443003e-05,0.1842157244682312,0.6806577108800411
30,0.1523809523809524,1.1331645011901856,9.646456421395446e-05,0.1463174968957901,0.7136855460703373
40,0.20317460317460317,1.0500896453857422,9.281359022841965e-05,0.15235942602157593,0.7293100193142891
50,0.25396825396825395,0.9814230918884277,8.800694230932884e-05,0.18486258387565613,0.7447394959628582
60,0.3047619047619048,0.9417718887329102,8.2174367055326e-05,0.20546093583106995,0.7523589514195919
70,0.35555555555555557,0.8714447975158691,7.547330408197695e-05,0.27494359016418457,0.7701701499521733
80,0.40634920634920635,0.8259673118591309,6.808463622961578e-05,0.2904492914676666,0.7822930864989758
90,0.45714285714285713,0.7809524536132812,6.0207806964560584e-05,0.6509429216384888,0.7934819318354129
100,0.5079365079365079,0.7385403633117675,5.205543677045049e-05,0.29676851630210876,0.804052384197712
110,0.5587301587301587,0.6973814964294434,4.384758385029457e-05,0.3625648319721222,0.8153482332825661
120,0.6095238095238096,0.6504372119903564,3.580580406099893e-05,0.3534400761127472,0.8266014747321606
130,0.6603174603174603,0.645383882522583,2.8147170421488272e-05,0.3655138909816742,0.8276743076741695
140,0.7111111111111111,0.6282196044921875,2.1078413626773546e-05,0.33809009194374084,0.8325855515897274
150,0.7619047619047619,0.602206802368164,1.4790341733986085e-05,0.3583182990550995,0.8397377617657185
160,0.8126984126984127,0.5957026004791259,9.452689650664515e-06,0.3788113296031952,0.8424368388950825
170,0.8634920634920635,0.5803478240966797,5.209537453863289e-06,0.34813615679740906,0.8462515577673912
180,0.9142857142857143,0.5834349155426025,2.1754212141102346e-06,0.3094688653945923,0.8446752935647964
190,0.9650793650793651,0.5862775802612304,4.322413053509944e-07,0.35028040409088135,0.8439742468297482
1 step epoch loss learning_rate grad_norm mean_token_accuracy
2 10 0.050793650793650794 1.5175626754760743 9.993914063644052e-05 0.2645105719566345 0.6436327129602433
3 20 0.10158730158730159 1.292062759399414 9.886131295443003e-05 0.1842157244682312 0.6806577108800411
4 30 0.1523809523809524 1.1331645011901856 9.646456421395446e-05 0.1463174968957901 0.7136855460703373
5 40 0.20317460317460317 1.0500896453857422 9.281359022841965e-05 0.15235942602157593 0.7293100193142891
6 50 0.25396825396825395 0.9814230918884277 8.800694230932884e-05 0.18486258387565613 0.7447394959628582
7 60 0.3047619047619048 0.9417718887329102 8.2174367055326e-05 0.20546093583106995 0.7523589514195919
8 70 0.35555555555555557 0.8714447975158691 7.547330408197695e-05 0.27494359016418457 0.7701701499521733
9 80 0.40634920634920635 0.8259673118591309 6.808463622961578e-05 0.2904492914676666 0.7822930864989758
10 90 0.45714285714285713 0.7809524536132812 6.0207806964560584e-05 0.6509429216384888 0.7934819318354129
11 100 0.5079365079365079 0.7385403633117675 5.205543677045049e-05 0.29676851630210876 0.804052384197712
12 110 0.5587301587301587 0.6973814964294434 4.384758385029457e-05 0.3625648319721222 0.8153482332825661
13 120 0.6095238095238096 0.6504372119903564 3.580580406099893e-05 0.3534400761127472 0.8266014747321606
14 130 0.6603174603174603 0.645383882522583 2.8147170421488272e-05 0.3655138909816742 0.8276743076741695
15 140 0.7111111111111111 0.6282196044921875 2.1078413626773546e-05 0.33809009194374084 0.8325855515897274
16 150 0.7619047619047619 0.602206802368164 1.4790341733986085e-05 0.3583182990550995 0.8397377617657185
17 160 0.8126984126984127 0.5957026004791259 9.452689650664515e-06 0.3788113296031952 0.8424368388950825
18 170 0.8634920634920635 0.5803478240966797 5.209537453863289e-06 0.34813615679740906 0.8462515577673912
19 180 0.9142857142857143 0.5834349155426025 2.1754212141102346e-06 0.3094688653945923 0.8446752935647964
20 190 0.9650793650793651 0.5862775802612304 4.322413053509944e-07 0.35028040409088135 0.8439742468297482

5
training/figures/training_v2_eval.csv Normal file
View File

@@ -0,0 +1,5 @@
step,epoch,eval_loss,eval_mean_token_accuracy
50,0.25396825396825395,0.9650015830993652,0.7496833843844277
100,0.5079365079365079,0.7067031860351562,0.8130695947578975
150,0.7619047619047619,0.5970866084098816,0.8415183879080272
197,1.0,0.575853168964386,0.8473851865246183
1 step epoch eval_loss eval_mean_token_accuracy
2 50 0.25396825396825395 0.9650015830993652 0.7496833843844277
3 100 0.5079365079365079 0.7067031860351562 0.8130695947578975
4 150 0.7619047619047619 0.5970866084098816 0.8415183879080272
5 197 1.0 0.575853168964386 0.8473851865246183

31
training/figures/training_v3.csv Normal file
View File

@@ -0,0 +1,31 @@
step,epoch,loss,learning_rate,grad_norm,mean_token_accuracy
10,0.06046863189720333,1.4868995666503906,9e-05,0.2206401377916336,0.6501498550176621
20,0.12093726379440665,1.280113124847412,9.980736554638366e-05,0.20303168892860413,0.6815041944384574
30,0.18140589569160998,1.1442659378051758,9.914337409040418e-05,0.12361272424459457,0.7108227916061878
40,0.2418745275888133,1.0783726692199707,9.801196139334195e-05,0.13824407756328583,0.7233529925346375
50,0.30234315948601664,1.0328060150146485,9.642388874981347e-05,0.1718689501285553,0.7331555418670177
60,0.36281179138321995,0.9452245712280274,9.439426092011875e-05,0.19833926856517792,0.7508241727948188
70,0.42328042328042326,0.8957448959350586,9.194238246314599e-05,0.27138301730155945,0.763477711379528
80,0.4837490551776266,0.8388309478759766,8.90915741234015e-05,0.8669945597648621,0.776527363806963
90,0.54421768707483,0.7785130500793457,8.586895101857747e-05,0.3251343071460724,0.7925561927258968
100,0.6046863189720333,0.7025893211364747,8.230516473739935e-05,0.45480582118034363,0.810952752828598
110,0.6651549508692366,0.6717377662658691,7.843411180075794e-05,0.3490012586116791,0.8204429857432842
120,0.7256235827664399,0.634728193283081,7.42926112590631e-05,0.4181080162525177,0.8306220307946205
130,0.7860922146636432,0.5837386608123779,6.992005449231208e-05,0.40689781308174133,0.843410138040781
140,0.8465608465608465,0.5329185962677002,6.535803054375738e-05,0.4665887653827667,0.8579231850802899
150,0.9070294784580499,0.5083013534545898,6.064993055076698e-05,0.4626559019088745,0.8642070218920708
160,0.9674981103552532,0.489164400100708,5.5840535035285025e-05,0.5883871912956238,0.8689255759119987
170,1.0241874527588812,0.43776092529296873,5.0975587979329734e-05,0.6238144040107727,0.8809238640467326
180,1.0846560846560847,0.4174932956695557,4.610136173665751e-05,0.4284566640853882,0.888404282182455
190,1.145124716553288,0.39289538860321044,4.1264216918883656e-05,0.42329925298690796,0.8944049634039402
200,1.2055933484504913,0.39194531440734864,3.651016144214878e-05,0.45279642939567566,0.8959574572741985
210,1.2660619803476947,0.3841248989105225,3.188441292840587e-05,0.4972575604915619,0.896124105155468
220,1.3265306122448979,0.35218961238861085,2.743096862349427e-05,0.5121544003486633,0.9058273300528527
230,1.3869992441421013,0.3745258331298828,2.3192186922673186e-05,0.38717055320739746,0.9008041150867939
240,1.4474678760393047,0.3568585395812988,1.9208384483883817e-05,0.40057459473609924,0.9051908120512963
250,1.507936507936508,0.34092986583709717,1.5517452760747975e-05,0.3971737325191498,0.9089121885597706
260,1.568405139833711,0.33079631328582765,1.2154497602603703e-05,0.36223524808883667,0.9112872503697872
270,1.6288737717309147,0.3347353458404541,9.151505349477902e-06,0.3942708671092987,0.9104152075946331
280,1.689342403628118,0.34016706943511965,6.53703859789348e-06,0.35229650139808655,0.9098294205963612
290,1.7498110355253211,0.3172227621078491,4.335964531197401e-06,0.38456934690475464,0.9160888768732548
300,1.8102796674225246,0.31731643676757815,2.5692183983629713e-06,0.5015138983726501,0.9155210055410862
1 step epoch loss learning_rate grad_norm mean_token_accuracy
2 10 0.06046863189720333 1.4868995666503906 9e-05 0.2206401377916336 0.6501498550176621
3 20 0.12093726379440665 1.280113124847412 9.980736554638366e-05 0.20303168892860413 0.6815041944384574
4 30 0.18140589569160998 1.1442659378051758 9.914337409040418e-05 0.12361272424459457 0.7108227916061878
5 40 0.2418745275888133 1.0783726692199707 9.801196139334195e-05 0.13824407756328583 0.7233529925346375
6 50 0.30234315948601664 1.0328060150146485 9.642388874981347e-05 0.1718689501285553 0.7331555418670177
7 60 0.36281179138321995 0.9452245712280274 9.439426092011875e-05 0.19833926856517792 0.7508241727948188
8 70 0.42328042328042326 0.8957448959350586 9.194238246314599e-05 0.27138301730155945 0.763477711379528
9 80 0.4837490551776266 0.8388309478759766 8.90915741234015e-05 0.8669945597648621 0.776527363806963
10 90 0.54421768707483 0.7785130500793457 8.586895101857747e-05 0.3251343071460724 0.7925561927258968
11 100 0.6046863189720333 0.7025893211364747 8.230516473739935e-05 0.45480582118034363 0.810952752828598
12 110 0.6651549508692366 0.6717377662658691 7.843411180075794e-05 0.3490012586116791 0.8204429857432842
13 120 0.7256235827664399 0.634728193283081 7.42926112590631e-05 0.4181080162525177 0.8306220307946205
14 130 0.7860922146636432 0.5837386608123779 6.992005449231208e-05 0.40689781308174133 0.843410138040781
15 140 0.8465608465608465 0.5329185962677002 6.535803054375738e-05 0.4665887653827667 0.8579231850802899
16 150 0.9070294784580499 0.5083013534545898 6.064993055076698e-05 0.4626559019088745 0.8642070218920708
17 160 0.9674981103552532 0.489164400100708 5.5840535035285025e-05 0.5883871912956238 0.8689255759119987
18 170 1.0241874527588812 0.43776092529296873 5.0975587979329734e-05 0.6238144040107727 0.8809238640467326
19 180 1.0846560846560847 0.4174932956695557 4.610136173665751e-05 0.4284566640853882 0.888404282182455
20 190 1.145124716553288 0.39289538860321044 4.1264216918883656e-05 0.42329925298690796 0.8944049634039402
21 200 1.2055933484504913 0.39194531440734864 3.651016144214878e-05 0.45279642939567566 0.8959574572741985
22 210 1.2660619803476947 0.3841248989105225 3.188441292840587e-05 0.4972575604915619 0.896124105155468
23 220 1.3265306122448979 0.35218961238861085 2.743096862349427e-05 0.5121544003486633 0.9058273300528527
24 230 1.3869992441421013 0.3745258331298828 2.3192186922673186e-05 0.38717055320739746 0.9008041150867939
25 240 1.4474678760393047 0.3568585395812988 1.9208384483883817e-05 0.40057459473609924 0.9051908120512963
26 250 1.507936507936508 0.34092986583709717 1.5517452760747975e-05 0.3971737325191498 0.9089121885597706
27 260 1.568405139833711 0.33079631328582765 1.2154497602603703e-05 0.36223524808883667 0.9112872503697872
28 270 1.6288737717309147 0.3347353458404541 9.151505349477902e-06 0.3942708671092987 0.9104152075946331
29 280 1.689342403628118 0.34016706943511965 6.53703859789348e-06 0.35229650139808655 0.9098294205963612
30 290 1.7498110355253211 0.3172227621078491 4.335964531197401e-06 0.38456934690475464 0.9160888768732548
31 300 1.8102796674225246 0.31731643676757815 2.5692183983629713e-06 0.5015138983726501 0.9155210055410862

7
training/figures/training_v3_eval.csv Normal file
View File

@@ -0,0 +1,7 @@
step,epoch,eval_loss
50,0.30234315948601664,1.009400725364685
100,0.6046863189720333,0.6973341703414917
150,0.9070294784580499,0.5049018859863281
200,1.2055933484504913,0.41107413172721863
250,1.507936507936508,0.3677152395248413
300,1.8102796674225246,0.35275769233703613
1 step epoch eval_loss
2 50 0.30234315948601664 1.009400725364685
3 100 0.6046863189720333 0.6973341703414917
4 150 0.9070294784580499 0.5049018859863281
5 200 1.2055933484504913 0.41107413172721863
6 250 1.507936507936508 0.3677152395248413
7 300 1.8102796674225246 0.35275769233703613

44
training/figures/training_v4.csv Normal file
View File

@@ -0,0 +1,44 @@
step,epoch,loss,eval_loss
10,0.056617126680820945,1.5663956642150878,
20,0.11323425336164189,1.3516300201416016,
30,0.16985138004246284,1.1845409393310546,
40,0.22646850672328378,1.1037321090698242,
50,0.28308563340410475,1.0377551078796388,
50,0.28308563340410475,,1.010231852531433
60,0.33970276008492567,0.9829194068908691,
70,0.39631988676574664,0.8914342880249023,
80,0.45293701344656756,0.79852294921875,
90,0.5095541401273885,0.7313865184783935,
100,0.5661712668082095,0.6939316749572754,
100,0.5661712668082095,,0.6673692464828491
110,0.6227883934890305,0.6318693161010742,
120,0.6794055201698513,0.5733372211456299,
130,0.7360226468506723,0.5655224800109864,
140,0.7926397735314933,0.49826998710632325,
150,0.8492569002123143,0.48117914199829104,
150,0.8492569002123143,,0.474189430475235
160,0.9058740268931351,0.4512802600860596,
170,0.9624911535739561,0.43169336318969725,
180,1.0169851380042463,0.39495258331298827,
190,1.0736022646850671,0.37260124683380125,
200,1.1302193913658882,0.3555767059326172,
200,1.1302193913658882,,0.3825647532939911
210,1.186836518046709,0.3555564880371094,
220,1.2434536447275302,0.3253514528274536,
230,1.300070771408351,0.3274040222167969,
240,1.356687898089172,0.31757657527923583,
250,1.413305024769993,0.32180213928222656,
250,1.413305024769993,,0.3373807370662689
260,1.4699221514508138,0.2948040008544922,
270,1.5265392781316347,0.2999974489212036,
280,1.5831564048124558,0.2915226936340332,
290,1.6397735314932769,0.28573825359344485,
300,1.6963906581740975,0.28896660804748536,
300,1.6963906581740975,,0.31564027070999146
310,1.7530077848549186,0.28694982528686525,
320,1.8096249115357397,0.27899909019470215,
330,1.8662420382165605,0.2718695878982544,
340,1.9228591648973814,0.28831920623779295,
350,1.9794762915782025,0.28850746154785156,
350,1.9794762915782025,,0.30873769521713257
354,2.0,,0.3087170422077179
1 step epoch loss eval_loss
2 10 0.056617126680820945 1.5663956642150878
3 20 0.11323425336164189 1.3516300201416016
4 30 0.16985138004246284 1.1845409393310546
5 40 0.22646850672328378 1.1037321090698242
6 50 0.28308563340410475 1.0377551078796388
7 50 0.28308563340410475 1.010231852531433
8 60 0.33970276008492567 0.9829194068908691
9 70 0.39631988676574664 0.8914342880249023
10 80 0.45293701344656756 0.79852294921875
11 90 0.5095541401273885 0.7313865184783935
12 100 0.5661712668082095 0.6939316749572754
13 100 0.5661712668082095 0.6673692464828491
14 110 0.6227883934890305 0.6318693161010742
15 120 0.6794055201698513 0.5733372211456299
16 130 0.7360226468506723 0.5655224800109864
17 140 0.7926397735314933 0.49826998710632325
18 150 0.8492569002123143 0.48117914199829104
19 150 0.8492569002123143 0.474189430475235
20 160 0.9058740268931351 0.4512802600860596
21 170 0.9624911535739561 0.43169336318969725
22 180 1.0169851380042463 0.39495258331298827
23 190 1.0736022646850671 0.37260124683380125
24 200 1.1302193913658882 0.3555767059326172
25 200 1.1302193913658882 0.3825647532939911
26 210 1.186836518046709 0.3555564880371094
27 220 1.2434536447275302 0.3253514528274536
28 230 1.300070771408351 0.3274040222167969
29 240 1.356687898089172 0.31757657527923583
30 250 1.413305024769993 0.32180213928222656
31 250 1.413305024769993 0.3373807370662689
32 260 1.4699221514508138 0.2948040008544922
33 270 1.5265392781316347 0.2999974489212036
34 280 1.5831564048124558 0.2915226936340332
35 290 1.6397735314932769 0.28573825359344485
36 300 1.6963906581740975 0.28896660804748536
37 300 1.6963906581740975 0.31564027070999146
38 310 1.7530077848549186 0.28694982528686525
39 320 1.8096249115357397 0.27899909019470215
40 330 1.8662420382165605 0.2718695878982544
41 340 1.9228591648973814 0.28831920623779295
42 350 1.9794762915782025 0.28850746154785156
43 350 1.9794762915782025 0.30873769521713257
44 354 2.0 0.3087170422077179

BIN
training/figures/v1_training_loss.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 48 KiB

BIN
training/figures/v2_vs_v1_training_loss.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 68 KiB

564
training/rag_eval_rescored.json Normal file
View File

@@ -0,0 +1,564 @@
{
"rescored_at": "2026-07-20 22:19:14",
"runs": {
"rag_dense_k8_base": {
"scored_total": 495,
"strict_correct": 17,
"strict_accuracy": 0.0343,
"lenient_correct": 29,
"lenient_accuracy": 0.0586,
"by_retrieval": {
"gold_in_context": {
"n": 94,
"strict": 0.1809,
"lenient": 0.2553
},
"gold_missing": {
"n": 38,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0138
}
}
},
"rag_dense_k8_legal-3b-v3-checkpoint-100": {
"scored_total": 495,
"strict_correct": 16,
"strict_accuracy": 0.0323,
"lenient_correct": 26,
"lenient_accuracy": 0.0525,
"by_retrieval": {
"gold_in_context": {
"n": 94,
"strict": 0.1702,
"lenient": 0.234
},
"gold_missing": {
"n": 38,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.011
}
}
},
"rag_dense_k8_legal-3b-v3-checkpoint-200": {
"scored_total": 495,
"strict_correct": 17,
"strict_accuracy": 0.0343,
"lenient_correct": 29,
"lenient_accuracy": 0.0586,
"by_retrieval": {
"gold_in_context": {
"n": 94,
"strict": 0.1809,
"lenient": 0.2553
},
"gold_missing": {
"n": 38,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0138
}
}
},
"rag_dense_k8_legal-3b-v3-checkpoint-300": {
"scored_total": 495,
"strict_correct": 19,
"strict_accuracy": 0.0384,
"lenient_correct": 27,
"lenient_accuracy": 0.0545,
"by_retrieval": {
"gold_in_context": {
"n": 94,
"strict": 0.2021,
"lenient": 0.266
},
"gold_missing": {
"n": 38,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0055
}
}
},
"rag_dense_k8_legal-3b-v3-checkpoint-332": {
"scored_total": 495,
"strict_correct": 15,
"strict_accuracy": 0.0303,
"lenient_correct": 25,
"lenient_accuracy": 0.0505,
"by_retrieval": {
"gold_in_context": {
"n": 94,
"strict": 0.1596,
"lenient": 0.234
},
"gold_missing": {
"n": 38,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0083
}
}
},
"rag_dense_k8_legal-3b-v3-final": {
"scored_total": 495,
"strict_correct": 15,
"strict_accuracy": 0.0303,
"lenient_correct": 25,
"lenient_accuracy": 0.0505,
"by_retrieval": {
"gold_in_context": {
"n": 94,
"strict": 0.1596,
"lenient": 0.234
},
"gold_missing": {
"n": 38,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0083
}
}
},
"rag_dense_k8_legal-3b-v4-checkpoint-100": {
"scored_total": 495,
"strict_correct": 19,
"strict_accuracy": 0.0384,
"lenient_correct": 34,
"lenient_accuracy": 0.0687,
"by_retrieval": {
"gold_in_context": {
"n": 98,
"strict": 0.1837,
"lenient": 0.2449
},
"gold_missing": {
"n": 52,
"strict": 0.0,
"lenient": 0.0192
},
"no_gold": {
"n": 345,
"strict": 0.0029,
"lenient": 0.0261
}
}
},
"rag_dense_k8_legal-3b-v4-checkpoint-200": {
"scored_total": 495,
"strict_correct": 18,
"strict_accuracy": 0.0364,
"lenient_correct": 28,
"lenient_accuracy": 0.0566,
"by_retrieval": {
"gold_in_context": {
"n": 98,
"strict": 0.1735,
"lenient": 0.2449
},
"gold_missing": {
"n": 52,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 345,
"strict": 0.0029,
"lenient": 0.0116
}
}
},
"rag_dense_k8_legal-3b-v4-checkpoint-300": {
"scored_total": 495,
"strict_correct": 17,
"strict_accuracy": 0.0343,
"lenient_correct": 28,
"lenient_accuracy": 0.0566,
"by_retrieval": {
"gold_in_context": {
"n": 98,
"strict": 0.1633,
"lenient": 0.2245
},
"gold_missing": {
"n": 52,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 345,
"strict": 0.0029,
"lenient": 0.0174
}
}
},
"rag_dense_k8_legal-3b-v4-checkpoint-354": {
"scored_total": 495,
"strict_correct": 19,
"strict_accuracy": 0.0384,
"lenient_correct": 30,
"lenient_accuracy": 0.0606,
"by_retrieval": {
"gold_in_context": {
"n": 98,
"strict": 0.1837,
"lenient": 0.2551
},
"gold_missing": {
"n": 52,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 345,
"strict": 0.0029,
"lenient": 0.0145
}
}
},
"rag_dense_k8_legal-3b-v4-final": {
"scored_total": 495,
"strict_correct": 19,
"strict_accuracy": 0.0384,
"lenient_correct": 30,
"lenient_accuracy": 0.0606,
"by_retrieval": {
"gold_in_context": {
"n": 98,
"strict": 0.1837,
"lenient": 0.2551
},
"gold_missing": {
"n": 52,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 345,
"strict": 0.0029,
"lenient": 0.0145
}
}
},
"rag_dense_k8_v3-dpo": {
"scored_total": 495,
"strict_correct": 17,
"strict_accuracy": 0.0343,
"lenient_correct": 24,
"lenient_accuracy": 0.0485,
"by_retrieval": {
"gold_in_context": {
"n": 94,
"strict": 0.1809,
"lenient": 0.234
},
"gold_missing": {
"n": 38,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0055
}
}
},
"rag_k8_base": {
"scored_total": 495,
"strict_correct": 19,
"strict_accuracy": 0.0384,
"lenient_correct": 29,
"lenient_accuracy": 0.0586,
"by_retrieval": {
"gold_in_context": {
"n": 85,
"strict": 0.2235,
"lenient": 0.2706
},
"gold_missing": {
"n": 47,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0165
}
}
},
"rag_k8_checkpoint-50": {
"scored_total": 495,
"strict_correct": 14,
"strict_accuracy": 0.0283,
"lenient_correct": 17,
"lenient_accuracy": 0.0343,
"by_retrieval": {
"gold_in_context": {
"n": 85,
"strict": 0.1647,
"lenient": 0.1765
},
"gold_missing": {
"n": 47,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0055
}
}
},
"rag_k8_legal-3b-v2-checkpoint-100": {
"scored_total": 495,
"strict_correct": 18,
"strict_accuracy": 0.0364,
"lenient_correct": 25,
"lenient_accuracy": 0.0505,
"by_retrieval": {
"gold_in_context": {
"n": 85,
"strict": 0.2118,
"lenient": 0.2824
},
"gold_missing": {
"n": 47,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0028
}
}
},
"rag_k8_legal-3b-v2-checkpoint-150": {
"scored_total": 495,
"strict_correct": 16,
"strict_accuracy": 0.0323,
"lenient_correct": 27,
"lenient_accuracy": 0.0545,
"by_retrieval": {
"gold_in_context": {
"n": 85,
"strict": 0.1882,
"lenient": 0.2588
},
"gold_missing": {
"n": 47,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0138
}
}
},
"rag_k8_legal-3b-v2-checkpoint-197": {
"scored_total": 495,
"strict_correct": 16,
"strict_accuracy": 0.0323,
"lenient_correct": 23,
"lenient_accuracy": 0.0465,
"by_retrieval": {
"gold_in_context": {
"n": 85,
"strict": 0.1882,
"lenient": 0.2471
},
"gold_missing": {
"n": 47,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0055
}
}
},
"rag_k8_legal-3b-v2-checkpoint-50": {
"scored_total": 495,
"strict_correct": 18,
"strict_accuracy": 0.0364,
"lenient_correct": 24,
"lenient_accuracy": 0.0485,
"by_retrieval": {
"gold_in_context": {
"n": 85,
"strict": 0.2118,
"lenient": 0.2588
},
"gold_missing": {
"n": 47,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0055
}
}
},
"rag_k8_legal-3b-v2-final": {
"scored_total": 495,
"strict_correct": 16,
"strict_accuracy": 0.0323,
"lenient_correct": 23,
"lenient_accuracy": 0.0465,
"by_retrieval": {
"gold_in_context": {
"n": 85,
"strict": 0.1882,
"lenient": 0.2471
},
"gold_missing": {
"n": 47,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 363,
"strict": 0.0,
"lenient": 0.0055
}
}
},
"rebase-r2_base": {
"scored_total": 495,
"strict_correct": 22,
"strict_accuracy": 0.0444,
"lenient_correct": 41,
"lenient_accuracy": 0.0828,
"by_retrieval": {
"gold_in_context": {
"n": 93,
"strict": 0.2151,
"lenient": 0.3011
},
"gold_missing": {
"n": 57,
"strict": 0.0175,
"lenient": 0.0526
},
"no_gold": {
"n": 345,
"strict": 0.0029,
"lenient": 0.029
}
}
},
"rebase-r2_v3-ckpt300": {
"scored_total": 495,
"strict_correct": 21,
"strict_accuracy": 0.0424,
"lenient_correct": 34,
"lenient_accuracy": 0.0687,
"by_retrieval": {
"gold_in_context": {
"n": 93,
"strict": 0.2151,
"lenient": 0.2903
},
"gold_missing": {
"n": 57,
"strict": 0.0,
"lenient": 0.0
},
"no_gold": {
"n": 345,
"strict": 0.0029,
"lenient": 0.0203
}
}
},
"rebase-r3_base": {
"scored_total": 495,
"strict_correct": 19,
"strict_accuracy": 0.0384,
"lenient_correct": 33,
"lenient_accuracy": 0.0667,
"by_retrieval": {
"gold_in_context": {
"n": 98,
"strict": 0.1735,
"lenient": 0.2449
},
"gold_missing": {
"n": 52,
"strict": 0.0192,
"lenient": 0.0192
},
"no_gold": {
"n": 345,
"strict": 0.0029,
"lenient": 0.0232
}
}
},
"rebase-r3_v3-ckpt300": {
"scored_total": 495,
"strict_correct": 21,
"strict_accuracy": 0.0424,
"lenient_correct": 32,
"lenient_accuracy": 0.0646,
"by_retrieval": {
"gold_in_context": {
"n": 98,
"strict": 0.1939,
"lenient": 0.2245
},
"gold_missing": {
"n": 52,
"strict": 0.0192,
"lenient": 0.0192
},
"no_gold": {
"n": 345,
"strict": 0.0029,
"lenient": 0.0261
}
}
},
"checkpoint-50": {
"scored_total": 495,
"strict_correct": 4,
"strict_accuracy": 0.0081,
"lenient_correct": 4,
"lenient_accuracy": 0.0081
}
}
}

91
training/retrieval_recall_dense.json Normal file
View File

@@ -0,0 +1,91 @@
{
"dense_model": "intfloat/multilingual-e5-base",
"eval_records": 500,
"gold_bearing": 150,
"no_citation_gold": 350,
"unresolved_fact_count": 0,
"unresolved_facts_sample": [],
"recall": {
"k=1": {
"any_hit": 0.4333,
"full_hit": 0.3867
},
"k=3": {
"any_hit": 0.5867,
"full_hit": 0.54
},
"k=5": {
"any_hit": 0.6267,
"full_hit": 0.58
},
"k=8": {
"any_hit": 0.7067,
"full_hit": 0.6533
}
},
"by_language": {
"english": {
"n": 139,
"full_hit@8": 0.6619
},
"hindi": {
"n": 5,
"full_hit@8": 0.8
},
"hinglish": {
"n": 6,
"full_hit@8": 0.3333
}
},
"by_domain": {
"bns": {
"n": 46,
"full_hit@8": 0.7174
},
"bnss": {
"n": 25,
"full_hit@8": 0.64
},
"bsa": {
"n": 11,
"full_hit@8": 0.3636
},
"cheque_bounce": {
"n": 8,
"full_hit@8": 0.5
},
"constitutional_law": {
"n": 33,
"full_hit@8": 0.7576
},
"cyber_law": {
"n": 12,
"full_hit@8": 0.5833
},
"labour_law": {
"n": 1,
"full_hit@8": 0.0
},
"motor_vehicles": {
"n": 4,
"full_hit@8": 0.75
},
"old_new_law_mapping": {
"n": 1,
"full_hit@8": 1.0
},
"rti": {
"n": 3,
"full_hit@8": 0.6667
},
"womens_protection": {
"n": 6,
"full_hit@8": 0.5
}
},
"phrase_coverage": {
"n": 350,
"any_fact_in_topk": 0.06
},
"measured_at": "2026-07-17 00:35:30"
}

39
training/train_v3.yaml Normal file
View File

@@ -0,0 +1,39 @@
# Nyaya-3B-v3 — RAFT: teacher answers regenerated UNDER the inference RAG
# prompt (data/splits_rag_v3, scripts/20+21), including deliberate
# retrieval-miss demonstrations. 2 epochs (v2's val loss was still falling
# at 1) + NEFTune noise for SFT robustness.
#
# Method: full-precision bf16 LoRA — NO quantization (project decision 2026-07-13).
run_name: legal-3b-v3
model_id: Qwen/Qwen2.5-3B-Instruct
output_dir: outputs/legal-3b-v3
data:
train_file: data/splits_rag_v3/train.jsonl
val_file: data/splits_rag_v3/val.jsonl
method: lora
lora:
r: 32
alpha: 64
dropout: 0.05
target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]
training:
max_seq_length: 6144
num_train_epochs: 2
learning_rate: 1.0e-4
warmup_ratio: 0.03
lr_scheduler_type: cosine
gradient_checkpointing: true
bf16: true
optim: adamw_torch_fused
neftune_noise_alpha: 5
per_device_train_batch_size: 4
gradient_accumulation_steps: 8
logging_steps: 10
# ~6.3K examples x 2 epochs / effective batch 32 = ~394 steps
eval_steps: 50
save_steps: 100

371
training/training_history_v3.json Normal file
View File

@@ -0,0 +1,371 @@
{
"run_name": "legal-3b-v3",
"log_history": [
{
"entropy": 1.2671179935336112,
"epoch": 0.06046863189720333,
"grad_norm": 0.2206401377916336,
"learning_rate": 9e-05,
"loss": 1.4868995666503906,
"mean_token_accuracy": 0.6501498550176621,
"num_tokens": 919743.0,
"step": 10
},
{
"entropy": 1.3823076233267784,
"epoch": 0.12093726379440665,
"grad_norm": 0.20303168892860413,
"learning_rate": 9.980736554638366e-05,
"loss": 1.280113124847412,
"mean_token_accuracy": 0.6815041944384574,
"num_tokens": 1841121.0,
"step": 20
},
{
"entropy": 1.1250081039965152,
"epoch": 0.18140589569160998,
"grad_norm": 0.12361272424459457,
"learning_rate": 9.914337409040418e-05,
"loss": 1.1442659378051758,
"mean_token_accuracy": 0.7108227916061878,
"num_tokens": 2765329.0,
"step": 30
},
{
"entropy": 1.0885243624448777,
"epoch": 0.2418745275888133,
"grad_norm": 0.13824407756328583,
"learning_rate": 9.801196139334195e-05,
"loss": 1.0783726692199707,
"mean_token_accuracy": 0.7233529925346375,
"num_tokens": 3693580.0,
"step": 40
},
{
"entropy": 1.0232380636036396,
"epoch": 0.30234315948601664,
"grad_norm": 0.1718689501285553,
"learning_rate": 9.642388874981347e-05,
"loss": 1.0328060150146485,
"mean_token_accuracy": 0.7331555418670177,
"num_tokens": 4603211.0,
"step": 50
},
{
"epoch": 0.30234315948601664,
"eval_entropy": 1.0007498400551933,
"eval_loss": 1.009400725364685,
"eval_mean_token_accuracy": 0.7387015887669155,
"eval_num_tokens": 4603211.0,
"eval_runtime": 89.0776,
"eval_samples_per_second": 3.143,
"eval_steps_per_second": 0.393,
"step": 50
},
{
"entropy": 0.9474448464810848,
"epoch": 0.36281179138321995,
"grad_norm": 0.19833926856517792,
"learning_rate": 9.439426092011875e-05,
"loss": 0.9452245712280274,
"mean_token_accuracy": 0.7508241727948188,
"num_tokens": 5511119.0,
"step": 60
},
{
"entropy": 0.9045603923499584,
"epoch": 0.42328042328042326,
"grad_norm": 0.27138301730155945,
"learning_rate": 9.194238246314599e-05,
"loss": 0.8957448959350586,
"mean_token_accuracy": 0.763477711379528,
"num_tokens": 6429339.0,
"step": 70
},
{
"entropy": 0.8463030077517033,
"epoch": 0.4837490551776266,
"grad_norm": 0.8669945597648621,
"learning_rate": 8.90915741234015e-05,
"loss": 0.8388309478759766,
"mean_token_accuracy": 0.776527363806963,
"num_tokens": 7370344.0,
"step": 80
},
{
"entropy": 0.7973637029528617,
"epoch": 0.54421768707483,
"grad_norm": 0.3251343071460724,
"learning_rate": 8.586895101857747e-05,
"loss": 0.7785130500793457,
"mean_token_accuracy": 0.7925561927258968,
"num_tokens": 8291090.0,
"step": 90
},
{
"entropy": 0.7309453509747982,
"epoch": 0.6046863189720333,
"grad_norm": 0.45480582118034363,
"learning_rate": 8.230516473739935e-05,
"loss": 0.7025893211364747,
"mean_token_accuracy": 0.810952752828598,
"num_tokens": 9207198.0,
"step": 100
},
{
"epoch": 0.6046863189720333,
"eval_entropy": 0.7214480527809688,
"eval_loss": 0.6973341703414917,
"eval_mean_token_accuracy": 0.8165236847741263,
"eval_num_tokens": 9207198.0,
"eval_runtime": 89.2073,
"eval_samples_per_second": 3.139,
"eval_steps_per_second": 0.392,
"step": 100
},
{
"entropy": 0.6975084710866213,
"epoch": 0.6651549508692366,
"grad_norm": 0.3490012586116791,
"learning_rate": 7.843411180075794e-05,
"loss": 0.6717377662658691,
"mean_token_accuracy": 0.8204429857432842,
"num_tokens": 10116405.0,
"step": 110
},
{
"entropy": 0.6599536526948213,
"epoch": 0.7256235827664399,
"grad_norm": 0.4181080162525177,
"learning_rate": 7.42926112590631e-05,
"loss": 0.634728193283081,
"mean_token_accuracy": 0.8306220307946205,
"num_tokens": 11029708.0,
"step": 120
},
{
"entropy": 0.6176572676748038,
"epoch": 0.7860922146636432,
"grad_norm": 0.40689781308174133,
"learning_rate": 6.992005449231208e-05,
"loss": 0.5837386608123779,
"mean_token_accuracy": 0.843410138040781,
"num_tokens": 11949565.0,
"step": 130
},
{
"entropy": 0.5673871148377657,
"epoch": 0.8465608465608465,
"grad_norm": 0.4665887653827667,
"learning_rate": 6.535803054375738e-05,
"loss": 0.5329185962677002,
"mean_token_accuracy": 0.8579231850802899,
"num_tokens": 12868154.0,
"step": 140
},
{
"entropy": 0.5449491806328297,
"epoch": 0.9070294784580499,
"grad_norm": 0.4626559019088745,
"learning_rate": 6.064993055076698e-05,
"loss": 0.5083013534545898,
"mean_token_accuracy": 0.8642070218920708,
"num_tokens": 13774405.0,
"step": 150
},
{
"epoch": 0.9070294784580499,
"eval_entropy": 0.5539889786924634,
"eval_loss": 0.5049018859863281,
"eval_mean_token_accuracy": 0.8679234419550215,
"eval_num_tokens": 13774405.0,
"eval_runtime": 89.3489,
"eval_samples_per_second": 3.134,
"eval_steps_per_second": 0.392,
"step": 150
},
{
"entropy": 0.5234581783413887,
"epoch": 0.9674981103552532,
"grad_norm": 0.5883871912956238,
"learning_rate": 5.5840535035285025e-05,
"loss": 0.489164400100708,
"mean_token_accuracy": 0.8689255759119987,
"num_tokens": 14698948.0,
"step": 160
},
{
"entropy": 0.49091413378715515,
"epoch": 1.0241874527588812,
"grad_norm": 0.6238144040107727,
"learning_rate": 5.0975587979329734e-05,
"loss": 0.43776092529296873,
"mean_token_accuracy": 0.8809238640467326,
"num_tokens": 15570380.0,
"step": 170
},
{
"entropy": 0.45481786243617534,
"epoch": 1.0846560846560847,
"grad_norm": 0.4284566640853882,
"learning_rate": 4.610136173665751e-05,
"loss": 0.4174932956695557,
"mean_token_accuracy": 0.888404282182455,
"num_tokens": 16489412.0,
"step": 180
},
{
"entropy": 0.4345656637102365,
"epoch": 1.145124716553288,
"grad_norm": 0.42329925298690796,
"learning_rate": 4.1264216918883656e-05,
"loss": 0.39289538860321044,
"mean_token_accuracy": 0.8944049634039402,
"num_tokens": 17402523.0,
"step": 190
},
{
"entropy": 0.4282470645383,
"epoch": 1.2055933484504913,
"grad_norm": 0.45279642939567566,
"learning_rate": 3.651016144214878e-05,
"loss": 0.39194531440734864,
"mean_token_accuracy": 0.8959574572741985,
"num_tokens": 18310900.0,
"step": 200
},
{
"epoch": 1.2055933484504913,
"eval_entropy": 0.4425007075071335,
"eval_loss": 0.41107413172721863,
"eval_mean_token_accuracy": 0.8935366255896432,
"eval_num_tokens": 18310900.0,
"eval_runtime": 89.1987,
"eval_samples_per_second": 3.139,
"eval_steps_per_second": 0.392,
"step": 200
},
{
"entropy": 0.42347441539168357,
"epoch": 1.2660619803476947,
"grad_norm": 0.4972575604915619,
"learning_rate": 3.188441292840587e-05,
"loss": 0.3841248989105225,
"mean_token_accuracy": 0.896124105155468,
"num_tokens": 19239874.0,
"step": 210
},
{
"entropy": 0.39230803996324537,
"epoch": 1.3265306122448979,
"grad_norm": 0.5121544003486633,
"learning_rate": 2.743096862349427e-05,
"loss": 0.35218961238861085,
"mean_token_accuracy": 0.9058273300528527,
"num_tokens": 20156230.0,
"step": 220
},
{
"entropy": 0.4154032004997134,
"epoch": 1.3869992441421013,
"grad_norm": 0.38717055320739746,
"learning_rate": 2.3192186922673186e-05,
"loss": 0.3745258331298828,
"mean_token_accuracy": 0.9008041150867939,
"num_tokens": 21063499.0,
"step": 230
},
{
"entropy": 0.39556570779532196,
"epoch": 1.4474678760393047,
"grad_norm": 0.40057459473609924,
"learning_rate": 1.9208384483883817e-05,
"loss": 0.3568585395812988,
"mean_token_accuracy": 0.9051908120512963,
"num_tokens": 21977605.0,
"step": 240
},
{
"entropy": 0.3834851888939738,
"epoch": 1.507936507936508,
"grad_norm": 0.3971737325191498,
"learning_rate": 1.5517452760747975e-05,
"loss": 0.34092986583709717,
"mean_token_accuracy": 0.9089121885597706,
"num_tokens": 22889175.0,
"step": 250
},
{
"epoch": 1.507936507936508,
"eval_entropy": 0.38803921937942504,
"eval_loss": 0.3677152395248413,
"eval_mean_token_accuracy": 0.9056272642953055,
"eval_num_tokens": 22889175.0,
"eval_runtime": 89.1071,
"eval_samples_per_second": 3.142,
"eval_steps_per_second": 0.393,
"step": 250
},
{
"entropy": 0.37403900306671856,
"epoch": 1.568405139833711,
"grad_norm": 0.36223524808883667,
"learning_rate": 1.2154497602603703e-05,
"loss": 0.33079631328582765,
"mean_token_accuracy": 0.9112872503697872,
"num_tokens": 23806399.0,
"step": 260
},
{
"entropy": 0.3777462661266327,
"epoch": 1.6288737717309147,
"grad_norm": 0.3942708671092987,
"learning_rate": 9.151505349477902e-06,
"loss": 0.3347353458404541,
"mean_token_accuracy": 0.9104152075946331,
"num_tokens": 24732235.0,
"step": 270
},
{
"entropy": 0.38299514539539814,
"epoch": 1.689342403628118,
"grad_norm": 0.35229650139808655,
"learning_rate": 6.53703859789348e-06,
"loss": 0.34016706943511965,
"mean_token_accuracy": 0.9098294205963612,
"num_tokens": 25644370.0,
"step": 280
},
{
"entropy": 0.3589464059099555,
"epoch": 1.7498110355253211,
"grad_norm": 0.38456934690475464,
"learning_rate": 4.335964531197401e-06,
"loss": 0.3172227621078491,
"mean_token_accuracy": 0.9160888768732548,
"num_tokens": 26579671.0,
"step": 290
},
{
"entropy": 0.35829958152025937,
"epoch": 1.8102796674225246,
"grad_norm": 0.5015138983726501,
"learning_rate": 2.5692183983629713e-06,
"loss": 0.31731643676757815,
"mean_token_accuracy": 0.9155210055410862,
"num_tokens": 27515269.0,
"step": 300
},
{
"epoch": 1.8102796674225246,
"eval_entropy": 0.374431825535638,
"eval_loss": 0.35275769233703613,
"eval_mean_token_accuracy": 0.9095644763537816,
"eval_num_tokens": 27515269.0,
"eval_runtime": 89.1755,
"eval_samples_per_second": 3.14,
"eval_steps_per_second": 0.392,
"step": 300
}
]
}