68 lines
2.4 KiB
JSON
68 lines
2.4 KiB
JSON
{
|
|
"benchmark_date": "2026-06-09",
|
|
"summary": "Local train-only Korean semantic sentinel diagnostics. These are not official leaderboard scores.",
|
|
"jumini_internal_scores_file": "jumini_internal_scores_20260609.json",
|
|
"main_public_comparison": {
|
|
"metric": "Korean semantic pass@33",
|
|
"prompt_set": "/tmp/chojm_hw2_runs/data/clean7020_passk32_accepted_imr_20260609/sentinel_imr_30x3_90.jsonl",
|
|
"decoding": {
|
|
"include_greedy_rollout": true,
|
|
"samples_per_prompt": 32,
|
|
"temperature": 0.7,
|
|
"top_p": 0.95,
|
|
"top_k": 50,
|
|
"max_new_tokens": 64
|
|
},
|
|
"models": {
|
|
"Jumini-Ko-1.2B-clean7020": {
|
|
"params": 1260505088,
|
|
"language_focus": "Korean from-scratch",
|
|
"summary_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_imr_k32_gpu2/passk_summary.json",
|
|
"pass_any": "38/90",
|
|
"pass_any_score": 42.22222222222222,
|
|
"pass_rollouts": "131/2884",
|
|
"pass_rollout_score": 4.542302357836339,
|
|
"greedy_hit": "0/90",
|
|
"categories": {
|
|
"instruction": "26/30",
|
|
"math": "8/30",
|
|
"reasoning": "4/30"
|
|
}
|
|
},
|
|
"TinyLlama-1.1B-intermediate-step-1431k-3T": {
|
|
"repo_id": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
|
|
"params": 1100000000,
|
|
"language_focus": "English general base",
|
|
"summary_json": "/tmp/chojm_hw2_runs/hf_baseline_compare_20260609/tinyllama_1p1b_base_k32/summary.json",
|
|
"pass_any": "31/90",
|
|
"pass_any_score": 34.44444444444444,
|
|
"pass_rollouts": "56/2966",
|
|
"pass_rollout_score": 1.8880647336480108,
|
|
"greedy_hit": "0/90",
|
|
"categories": {
|
|
"instruction": "27/30",
|
|
"math": "4/30",
|
|
"reasoning": "0/30"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"public_refresh_evidence": {
|
|
"metric": "Stratified Korean semantic pass@k",
|
|
"previous_hf_v3a6200": {
|
|
"summary_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/v3a6200_strat240/passk_summary.json",
|
|
"pass_any": "0/240",
|
|
"pass_any_score": 0.0,
|
|
"pass_rollouts": "0/2094",
|
|
"pass_rollout_score": 0.0
|
|
},
|
|
"current_clean7020": {
|
|
"summary_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_strat240_gpu2/passk_summary.json",
|
|
"pass_any": "21/240",
|
|
"pass_any_score": 8.75,
|
|
"pass_rollouts": "29/2105",
|
|
"pass_rollout_score": 1.3776722090261282
|
|
}
|
|
}
|
|
}
|