{ "created_at": "2026-06-09T10:22:09.271056+00:00", "repo_id": "properly59/Jumini-Ko-1.2B", "model_card_scope": "Completed score summaries for the current public clean7020 checkpoint. Row-level prompts/completions and non-exported descendant checkpoints are intentionally omitted.", "current_public_checkpoint": { "name": "clean7020", "native_checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "export_dir": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32", "source_training_step": 7020, "tokens_seen": 1025474560, "hf_remote_sha_before_this_appendix": "82815c2b339f8674792b8a5910c3eb913aab9e60" }, "validation": { "transformers_load_smoke": { "source_json": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32/load_smoke.json", "ok": true, "model_class": "LlamaForCausalLM", "parameter_count": 1260505088, "tokenizer_len": 128000 }, "hf_export_equivalence": { "source_json": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32/hf_equivalence_atol2e-5.json", "ok": true, "atol": 2e-05, "rtol": 2e-05, "max_abs_diff": 1.4781951904296875e-05, "mean_abs_diff": 1.073120984074194e-06, "rms_diff": 1.5380918512164499e-06, "argmax_mismatch_rate": 0.0 }, "native_checkpoint_summary": { "source_json": "/tmp/chojm_hw2_runs/eval/argmax_boundary_layernormshock_rerun20260608_gate/step_00007020/checkpoint-summary.json", "ok": true, "step": 7020, "tokens_seen": 1025474560, "file_size_bytes": 6090713562, "native_tensor_parameter_count": 1522649088 } }, "strict_heldout_project_gate": { "source_json": "/tmp/chojm_hw2_runs/eval/argmax_boundary_layernormshock_rerun20260608_gate/step_00007020/heldout-semantic-gate.json", "preset": "greedy|plain", "rows": 12, "semantic_hits": { "count": 2, "total": 12, "rate": 0.16666666666666666, "percent": 16.6667 }, "stable_semantic_hits": { "count": 2, "total": 12, "rate": 0.16666666666666666, "percent": 16.6667 }, "eos_hits": { "count": 5, "total": 12, "rate": 0.4166666666666667, "percent": 41.6667 }, "max_new_tokens_hits": { "count": 7, "total": 12, "rate": 0.5833333333333334, "percent": 58.3333 }, "mechanical_bad": { "count": 9, "total": 12, "rate": 0.75, "percent": 75.0 }, "empty": { "count": 1, "total": 12, "rate": 0.08333333333333333, "percent": 8.3333 }, "semantic_hit_ids": [ "heldout_summary_battery_001", "heldout_minutes_split_001" ], "flag_counts": { "max_new_tokens_hit": 7, "token_3gram_repetition": 8, "punctuation_heavy": 3, "char_loop": 3, "low_hangul_alnum": 3, "empty": 1 } }, "korean_semantic_passk_and_decode_diagnostics": { "k32_korean_semantic_sentinel_90": { "label": "90-prompt Korean semantic sentinel, greedy + 32 sampled rollouts", "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_imr_k32_gpu2/passk_summary.json", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "settings": { "eos_logit_bias": 0.0, "include_greedy_rollout": true, "max_new_tokens": 64, "no_repeat_ngram_size": 0, "precision": "fp16", "prompt_variant": "plain", "repetition_penalty": 1.0, "samples_per_prompt": 32, "seed": 20260609, "temperature": 0.7, "top_k": 50, "top_p": 0.95 }, "totals": { "evaluated_prompts": 90, "read_prompts": 90, "rollouts": 2884, "pass_any_prompts": 38, "pass_any_rate": 0.4222222222222222, "pass_rollouts": 131, "pass_rollout_rate": 0.045423023578363385, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 38, "sample_hit_rate": 0.4222222222222222, "eos_hits": 2071, "eos_hit_rate": 0.7180998613037448, "mechanical_bad_rollouts": 930, "mechanical_bad_rate": 0.3224687933425798, "pass_any_categories": [ "instruction", "math", "reasoning" ], "mechanical_flags": { "char_loop": 222, "punctuation_heavy": 226, "role_leak": 8, "token_3gram_repetition": 914 } }, "by_category": { "instruction": { "rollouts": 913, "pass_any_prompts": 26, "pass_any_rate": 0.8666666666666667, "pass_rollouts": 110, "pass_rollout_rate": 0.12048192771084337, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 26, "sample_hit_rate": 0.8666666666666667, "eos_hits": 561, "eos_hit_rate": 0.6144578313253012, "mechanical_bad_rollouts": 384, "mechanical_bad_rate": 0.42059145673603504, "mechanical_flags": { "char_loop": 152, "punctuation_heavy": 161, "token_3gram_repetition": 373 }, "prompts": 30 }, "math": { "rollouts": 986, "pass_any_prompts": 8, "pass_any_rate": 0.26666666666666666, "pass_rollouts": 14, "pass_rollout_rate": 0.014198782961460446, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 8, "sample_hit_rate": 0.26666666666666666, "eos_hits": 743, "eos_hit_rate": 0.7535496957403651, "mechanical_bad_rollouts": 311, "mechanical_bad_rate": 0.3154158215010142, "mechanical_flags": { "char_loop": 18, "punctuation_heavy": 15, "role_leak": 8, "token_3gram_repetition": 306 }, "prompts": 30 }, "reasoning": { "rollouts": 985, "pass_any_prompts": 4, "pass_any_rate": 0.13333333333333333, "pass_rollouts": 7, "pass_rollout_rate": 0.007106598984771574, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 4, "sample_hit_rate": 0.13333333333333333, "eos_hits": 767, "eos_hit_rate": 0.7786802030456853, "mechanical_bad_rollouts": 235, "mechanical_bad_rate": 0.23857868020304568, "mechanical_flags": { "char_loop": 52, "punctuation_heavy": 50, "token_3gram_repetition": 235 }, "prompts": 30 } } }, "stratified_korean_semantic_sentinel_240": { "label": "240-prompt stratified Korean semantic sentinel", "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_strat240_gpu2/passk_summary.json", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "settings": { "eos_logit_bias": 0.0, "include_greedy_rollout": true, "max_new_tokens": 64, "no_repeat_ngram_size": 0, "precision": "fp16", "prompt_variant": "plain", "repetition_penalty": 1.0, "samples_per_prompt": 8, "seed": 20260609, "temperature": 0.7, "top_k": 50, "top_p": 0.95 }, "totals": { "evaluated_prompts": 240, "read_prompts": 240, "rollouts": 2105, "pass_any_prompts": 21, "pass_any_rate": 0.0875, "pass_rollouts": 29, "pass_rollout_rate": 0.013776722090261283, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 21, "sample_hit_rate": 0.0875, "eos_hits": 1238, "eos_hit_rate": 0.58812351543943, "mechanical_bad_rollouts": 856, "mechanical_bad_rate": 0.4066508313539192, "pass_any_categories": [ "instruction", "math", "reasoning" ], "mechanical_flags": { "char_loop": 230, "punctuation_heavy": 214, "role_leak": 17, "token_3gram_repetition": 838 } }, "by_category": { "causal": { "rollouts": 262, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 122, "eos_hit_rate": 0.46564885496183206, "mechanical_bad_rollouts": 156, "mechanical_bad_rate": 0.5954198473282443, "mechanical_flags": { "char_loop": 37, "punctuation_heavy": 35, "token_3gram_repetition": 156 }, "prompts": 30 }, "compare": { "rollouts": 265, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 155, "eos_hit_rate": 0.5849056603773585, "mechanical_bad_rollouts": 124, "mechanical_bad_rate": 0.4679245283018868, "mechanical_flags": { "char_loop": 25, "punctuation_heavy": 25, "role_leak": 3, "token_3gram_repetition": 122 }, "prompts": 30 }, "instruction": { "rollouts": 255, "pass_any_prompts": 17, "pass_any_rate": 0.5666666666666667, "pass_rollouts": 24, "pass_rollout_rate": 0.09411764705882353, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 17, "sample_hit_rate": 0.5666666666666667, "eos_hits": 149, "eos_hit_rate": 0.5843137254901961, "mechanical_bad_rollouts": 119, "mechanical_bad_rate": 0.4666666666666667, "mechanical_flags": { "char_loop": 64, "punctuation_heavy": 66, "token_3gram_repetition": 116 }, "prompts": 30 }, "math": { "rollouts": 270, "pass_any_prompts": 2, "pass_any_rate": 0.06666666666666667, "pass_rollouts": 2, "pass_rollout_rate": 0.007407407407407408, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 2, "sample_hit_rate": 0.06666666666666667, "eos_hits": 207, "eos_hit_rate": 0.7666666666666667, "mechanical_bad_rollouts": 92, "mechanical_bad_rate": 0.34074074074074073, "mechanical_flags": { "char_loop": 4, "punctuation_heavy": 3, "role_leak": 4, "token_3gram_repetition": 90 }, "prompts": 30 }, "notice": { "rollouts": 270, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 71, "eos_hit_rate": 0.26296296296296295, "mechanical_bad_rollouts": 99, "mechanical_bad_rate": 0.36666666666666664, "mechanical_flags": { "char_loop": 19, "punctuation_heavy": 8, "token_3gram_repetition": 96 }, "prompts": 30 }, "reasoning": { "rollouts": 269, "pass_any_prompts": 2, "pass_any_rate": 0.06666666666666667, "pass_rollouts": 3, "pass_rollout_rate": 0.011152416356877323, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 2, "sample_hit_rate": 0.06666666666666667, "eos_hits": 207, "eos_hit_rate": 0.7695167286245354, "mechanical_bad_rollouts": 59, "mechanical_bad_rate": 0.21933085501858737, "mechanical_flags": { "char_loop": 7, "punctuation_heavy": 9, "token_3gram_repetition": 59 }, "prompts": 30 }, "safety": { "rollouts": 258, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 156, "eos_hit_rate": 0.6046511627906976, "mechanical_bad_rollouts": 111, "mechanical_bad_rate": 0.43023255813953487, "mechanical_flags": { "char_loop": 30, "punctuation_heavy": 29, "role_leak": 9, "token_3gram_repetition": 104 }, "prompts": 30 }, "style": { "rollouts": 256, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 171, "eos_hit_rate": 0.66796875, "mechanical_bad_rollouts": 96, "mechanical_bad_rate": 0.375, "mechanical_flags": { "char_loop": 44, "punctuation_heavy": 39, "role_leak": 1, "token_3gram_repetition": 95 }, "prompts": 30 } } }, "k3_plain64_baseline_90": { "label": "90-prompt Korean semantic sentinel, greedy + 3 sampled rollouts", "source_json": "/tmp/chojm_hw2_runs/checkpoint_interpolation_20260609/clean7020_baseline/passk_imr_k3_plain64/passk_summary.json", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "settings": { "eos_logit_bias": 0.0, "include_greedy_rollout": true, "max_new_tokens": 64, "no_repeat_ngram_size": 0, "precision": "fp16", "prompt_variant": "plain", "repetition_penalty": 1.0, "samples_per_prompt": 3, "seed": 20260609, "temperature": 0.7, "top_k": 50, "top_p": 0.95 }, "totals": { "evaluated_prompts": 90, "read_prompts": 90, "rollouts": 357, "pass_any_prompts": 11, "pass_any_rate": 0.12222222222222222, "pass_rollouts": 11, "pass_rollout_rate": 0.03081232492997199, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 11, "sample_hit_rate": 0.12222222222222222, "eos_hits": 224, "eos_hit_rate": 0.6274509803921569, "mechanical_bad_rollouts": 163, "mechanical_bad_rate": 0.4565826330532213, "pass_any_categories": [ "instruction", "math" ], "mechanical_flags": { "char_loop": 56, "punctuation_heavy": 56, "token_3gram_repetition": 161 } }, "by_category": { "instruction": { "rollouts": 119, "pass_any_prompts": 10, "pass_any_rate": 0.3333333333333333, "pass_rollouts": 10, "pass_rollout_rate": 0.08403361344537816, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 10, "sample_hit_rate": 0.3333333333333333, "eos_hits": 58, "eos_hit_rate": 0.48739495798319327, "mechanical_bad_rollouts": 67, "mechanical_bad_rate": 0.5630252100840336, "mechanical_flags": { "char_loop": 44, "punctuation_heavy": 45, "token_3gram_repetition": 65 }, "prompts": 30 }, "math": { "rollouts": 119, "pass_any_prompts": 1, "pass_any_rate": 0.03333333333333333, "pass_rollouts": 1, "pass_rollout_rate": 0.008403361344537815, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 1, "sample_hit_rate": 0.03333333333333333, "eos_hits": 73, "eos_hit_rate": 0.6134453781512605, "mechanical_bad_rollouts": 61, "mechanical_bad_rate": 0.5126050420168067, "mechanical_flags": { "char_loop": 5, "punctuation_heavy": 3, "token_3gram_repetition": 61 }, "prompts": 30 }, "reasoning": { "rollouts": 119, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 93, "eos_hit_rate": 0.7815126050420168, "mechanical_bad_rollouts": 35, "mechanical_bad_rate": 0.29411764705882354, "mechanical_flags": { "char_loop": 7, "punctuation_heavy": 8, "token_3gram_repetition": 35 }, "prompts": 30 } } }, "greedy_only_90": { "label": "90-prompt Korean semantic sentinel, greedy only", "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_sentinel_imr_greedy_gpu1/passk_summary.json", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "settings": { "eos_logit_bias": 0.0, "include_greedy_rollout": true, "max_new_tokens": 64, "no_repeat_ngram_size": 0, "precision": "fp16", "prompt_variant": "plain", "repetition_penalty": 1.0, "samples_per_prompt": 0, "seed": 20260610, "temperature": 0.7, "top_k": 50, "top_p": 0.95 }, "totals": { "evaluated_prompts": 90, "read_prompts": 90, "rollouts": 90, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 46, "eos_hit_rate": 0.5111111111111111, "mechanical_bad_rollouts": 65, "mechanical_bad_rate": 0.7222222222222222, "pass_any_categories": [], "mechanical_flags": { "char_loop": 24, "punctuation_heavy": 24, "token_3gram_repetition": 65 } }, "by_category": { "instruction": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 8, "eos_hit_rate": 0.26666666666666666, "mechanical_bad_rollouts": 22, "mechanical_bad_rate": 0.7333333333333333, "mechanical_flags": { "char_loop": 22, "punctuation_heavy": 22, "token_3gram_repetition": 22 }, "prompts": 30 }, "math": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 15, "eos_hit_rate": 0.5, "mechanical_bad_rollouts": 27, "mechanical_bad_rate": 0.9, "mechanical_flags": { "char_loop": 1, "punctuation_heavy": 1, "token_3gram_repetition": 27 }, "prompts": 30 }, "reasoning": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 23, "eos_hit_rate": 0.7666666666666667, "mechanical_bad_rollouts": 16, "mechanical_bad_rate": 0.5333333333333333, "mechanical_flags": { "char_loop": 1, "punctuation_heavy": 1, "token_3gram_repetition": 16 }, "prompts": 30 } } }, "two_prompt_smoke": { "label": "2-prompt generation smoke only", "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/smoke_clean7020/summary.json", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "settings": { "eos_logit_bias": 0.0, "include_greedy_rollout": true, "max_new_tokens": 16, "no_repeat_ngram_size": 0, "precision": "fp16", "prompt_variant": "plain", "repetition_penalty": 1.0, "samples_per_prompt": 1, "seed": 20260609, "temperature": 0.7, "top_k": 50, "top_p": 0.95 }, "totals": { "evaluated_prompts": 2, "read_prompts": 2, "rollouts": 4, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 0, "eos_hit_rate": 0.0, "mechanical_bad_rollouts": 1, "mechanical_bad_rate": 0.25, "pass_any_categories": [], "mechanical_flags": { "token_3gram_repetition": 1 } }, "by_category": { "compare": { "rollouts": 2, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 0, "eos_hit_rate": 0.0, "mechanical_bad_rollouts": 1, "mechanical_bad_rate": 0.5, "mechanical_flags": { "token_3gram_repetition": 1 }, "prompts": 1 }, "notice": { "rollouts": 2, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 0, "eos_hit_rate": 0.0, "mechanical_bad_rollouts": 0, "mechanical_bad_rate": 0.0, "mechanical_flags": {}, "prompts": 1 } } }, "decode_control_eosbias1": { "label": "90-prompt greedy decode control: eosbias1", "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_decode_sentinel/eosbias1/passk_summary.json", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "settings": { "eos_logit_bias": 1.0, "include_greedy_rollout": true, "max_new_tokens": 64, "no_repeat_ngram_size": 0, "precision": "fp16", "prompt_variant": "plain", "repetition_penalty": 1.0, "samples_per_prompt": 0, "seed": 20260610, "temperature": 0.7, "top_k": 50, "top_p": 0.95 }, "totals": { "evaluated_prompts": 90, "read_prompts": 90, "rollouts": 90, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 46, "eos_hit_rate": 0.5111111111111111, "mechanical_bad_rollouts": 65, "mechanical_bad_rate": 0.7222222222222222, "pass_any_categories": [], "mechanical_flags": { "char_loop": 24, "punctuation_heavy": 24, "token_3gram_repetition": 65 } }, "by_category": { "instruction": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 8, "eos_hit_rate": 0.26666666666666666, "mechanical_bad_rollouts": 22, "mechanical_bad_rate": 0.7333333333333333, "mechanical_flags": { "char_loop": 22, "punctuation_heavy": 22, "token_3gram_repetition": 22 }, "prompts": 30 }, "math": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 15, "eos_hit_rate": 0.5, "mechanical_bad_rollouts": 27, "mechanical_bad_rate": 0.9, "mechanical_flags": { "char_loop": 1, "punctuation_heavy": 1, "token_3gram_repetition": 27 }, "prompts": 30 }, "reasoning": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 23, "eos_hit_rate": 0.7666666666666667, "mechanical_bad_rollouts": 16, "mechanical_bad_rate": 0.5333333333333333, "mechanical_flags": { "char_loop": 1, "punctuation_heavy": 1, "token_3gram_repetition": 16 }, "prompts": 30 } } }, "decode_control_no_repeat3": { "label": "90-prompt greedy decode control: no_repeat3", "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_decode_sentinel/no_repeat3/passk_summary.json", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "settings": { "eos_logit_bias": 0.0, "include_greedy_rollout": true, "max_new_tokens": 64, "no_repeat_ngram_size": 3, "precision": "fp16", "prompt_variant": "plain", "repetition_penalty": 1.0, "samples_per_prompt": 0, "seed": 20260610, "temperature": 0.7, "top_k": 50, "top_p": 0.95 }, "totals": { "evaluated_prompts": 90, "read_prompts": 90, "rollouts": 90, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 46, "eos_hit_rate": 0.5111111111111111, "mechanical_bad_rollouts": 65, "mechanical_bad_rate": 0.7222222222222222, "pass_any_categories": [], "mechanical_flags": { "char_loop": 24, "punctuation_heavy": 24, "token_3gram_repetition": 65 } }, "by_category": { "instruction": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 8, "eos_hit_rate": 0.26666666666666666, "mechanical_bad_rollouts": 22, "mechanical_bad_rate": 0.7333333333333333, "mechanical_flags": { "char_loop": 22, "punctuation_heavy": 22, "token_3gram_repetition": 22 }, "prompts": 30 }, "math": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 15, "eos_hit_rate": 0.5, "mechanical_bad_rollouts": 27, "mechanical_bad_rate": 0.9, "mechanical_flags": { "char_loop": 1, "punctuation_heavy": 1, "token_3gram_repetition": 27 }, "prompts": 30 }, "reasoning": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 23, "eos_hit_rate": 0.7666666666666667, "mechanical_bad_rollouts": 16, "mechanical_bad_rate": 0.5333333333333333, "mechanical_flags": { "char_loop": 1, "punctuation_heavy": 1, "token_3gram_repetition": 16 }, "prompts": 30 } } }, "decode_control_rep108": { "label": "90-prompt greedy decode control: rep108", "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_decode_sentinel/rep108/passk_summary.json", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "settings": { "eos_logit_bias": 0.0, "include_greedy_rollout": true, "max_new_tokens": 64, "no_repeat_ngram_size": 0, "precision": "fp16", "prompt_variant": "plain", "repetition_penalty": 1.08, "samples_per_prompt": 0, "seed": 20260610, "temperature": 0.7, "top_k": 50, "top_p": 0.95 }, "totals": { "evaluated_prompts": 90, "read_prompts": 90, "rollouts": 90, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 46, "eos_hit_rate": 0.5111111111111111, "mechanical_bad_rollouts": 65, "mechanical_bad_rate": 0.7222222222222222, "pass_any_categories": [], "mechanical_flags": { "char_loop": 24, "punctuation_heavy": 24, "token_3gram_repetition": 65 } }, "by_category": { "instruction": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 8, "eos_hit_rate": 0.26666666666666666, "mechanical_bad_rollouts": 22, "mechanical_bad_rate": 0.7333333333333333, "mechanical_flags": { "char_loop": 22, "punctuation_heavy": 22, "token_3gram_repetition": 22 }, "prompts": 30 }, "math": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 15, "eos_hit_rate": 0.5, "mechanical_bad_rollouts": 27, "mechanical_bad_rate": 0.9, "mechanical_flags": { "char_loop": 1, "punctuation_heavy": 1, "token_3gram_repetition": 27 }, "prompts": 30 }, "reasoning": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 23, "eos_hit_rate": 0.7666666666666667, "mechanical_bad_rollouts": 16, "mechanical_bad_rate": 0.5333333333333333, "mechanical_flags": { "char_loop": 1, "punctuation_heavy": 1, "token_3gram_repetition": 16 }, "prompts": 30 } } }, "decode_control_rep108_eos1": { "label": "90-prompt greedy decode control: rep108_eos1", "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_decode_sentinel/rep108_eos1/passk_summary.json", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "settings": { "eos_logit_bias": 1.0, "include_greedy_rollout": true, "max_new_tokens": 64, "no_repeat_ngram_size": 0, "precision": "fp16", "prompt_variant": "plain", "repetition_penalty": 1.08, "samples_per_prompt": 0, "seed": 20260610, "temperature": 0.7, "top_k": 50, "top_p": 0.95 }, "totals": { "evaluated_prompts": 90, "read_prompts": 90, "rollouts": 90, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 46, "eos_hit_rate": 0.5111111111111111, "mechanical_bad_rollouts": 65, "mechanical_bad_rate": 0.7222222222222222, "pass_any_categories": [], "mechanical_flags": { "char_loop": 24, "punctuation_heavy": 24, "token_3gram_repetition": 65 } }, "by_category": { "instruction": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 8, "eos_hit_rate": 0.26666666666666666, "mechanical_bad_rollouts": 22, "mechanical_bad_rate": 0.7333333333333333, "mechanical_flags": { "char_loop": 22, "punctuation_heavy": 22, "token_3gram_repetition": 22 }, "prompts": 30 }, "math": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 15, "eos_hit_rate": 0.5, "mechanical_bad_rollouts": 27, "mechanical_bad_rate": 0.9, "mechanical_flags": { "char_loop": 1, "punctuation_heavy": 1, "token_3gram_repetition": 27 }, "prompts": 30 }, "reasoning": { "rollouts": 30, "pass_any_prompts": 0, "pass_any_rate": 0.0, "pass_rollouts": 0, "pass_rollout_rate": 0.0, "greedy_hit_prompts": 0, "greedy_hit_rate": 0.0, "sample_hit_prompts": 0, "sample_hit_rate": 0.0, "eos_hits": 23, "eos_hit_rate": 0.7666666666666667, "mechanical_bad_rollouts": 16, "mechanical_bad_rate": 0.5333333333333333, "mechanical_flags": { "char_loop": 1, "punctuation_heavy": 1, "token_3gram_repetition": 16 }, "prompts": 30 } } } }, "korean_public_proxy_stagegate": { "belebele_kor": { "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/belebele_kor.json", "accuracy": 0.275, "accuracy_percent": 27.5, "correct": 55, "total": 200, "score": "avg_logprob", "categories": { "belebele:kor_Hang": { "accuracy": 0.275, "correct": 55, "total": 200, "accuracy_percent": 27.5 } } }, "gsm8k_ko": { "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/gsm8k_ko.json", "accuracy": 0.0, "accuracy_percent": 0.0, "correct": 0, "total": 100, "max_new_tokens": 192 }, "haerae_bench_2_0": { "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/haerae_bench_2_0.json", "accuracy": 0.16666666666666666, "accuracy_percent": 16.6667, "correct": 10, "total": 60, "score": "avg_logprob", "categories": { "haerae-bench-2.0:context_definition_alignment": { "accuracy": 0.26666666666666666, "correct": 8, "total": 30, "accuracy_percent": 26.6667 }, "haerae-bench-2.0:date_understanding": { "accuracy": 0.06666666666666667, "correct": 2, "total": 30, "accuracy_percent": 6.6667 } } }, "hrm8k": { "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/hrm8k.json", "accuracy": 0.03333333333333333, "accuracy_percent": 3.3333, "correct": 2, "total": 60, "max_new_tokens": 192 }, "kmmlu_hard": { "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/kmmlu_hard.json", "accuracy": 0.2125, "accuracy_percent": 21.25, "correct": 34, "total": 160, "score": "avg_logprob", "categories": { "Biology": { "accuracy": 0.2, "correct": 4, "total": 20, "accuracy_percent": 20.0 }, "Computer Science": { "accuracy": 0.25, "correct": 5, "total": 20, "accuracy_percent": 25.0 }, "Education": { "accuracy": 0.1, "correct": 2, "total": 20, "accuracy_percent": 10.0 }, "Environmental Science": { "accuracy": 0.2, "correct": 4, "total": 20, "accuracy_percent": 20.0 }, "Fashion": { "accuracy": 0.2, "correct": 4, "total": 20, "accuracy_percent": 20.0 }, "Korean History": { "accuracy": 0.3, "correct": 6, "total": 20, "accuracy_percent": 30.0 }, "Psychology": { "accuracy": 0.2, "correct": 4, "total": 20, "accuracy_percent": 20.0 }, "Railway and Automotive Engineering": { "accuracy": 0.25, "correct": 5, "total": 20, "accuracy_percent": 25.0 } } }, "kobalt": { "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/kobalt.json", "accuracy": 0.095, "accuracy_percent": 9.5, "correct": 19, "total": 200, "score": "avg_logprob", "categories": { "kobalt:Phonetics/Phonology:Phonological Alternation": { "accuracy": 0.1111111111111111, "correct": 1, "total": 9, "accuracy_percent": 11.1111 }, "kobalt:Phonetics/Phonology:Phonological Constraints": { "accuracy": 0.0, "correct": 0, "total": 3, "accuracy_percent": 0.0 }, "kobalt:Pragmatics:Deixis & Reference": { "accuracy": 0.0, "correct": 0, "total": 12, "accuracy_percent": 0.0 }, "kobalt:Pragmatics:Discourse Principles": { "accuracy": 0.0, "correct": 0, "total": 6, "accuracy_percent": 0.0 }, "kobalt:Pragmatics:Implicature": { "accuracy": 0.14285714285714285, "correct": 1, "total": 7, "accuracy_percent": 14.2857 }, "kobalt:Pragmatics:Social Relationship Marking": { "accuracy": 0.5, "correct": 1, "total": 2, "accuracy_percent": 50.0 }, "kobalt:Pragmatics:Speech Acts": { "accuracy": 0.05555555555555555, "correct": 1, "total": 18, "accuracy_percent": 5.5556 }, "kobalt:Semantics:Conjunctions": { "accuracy": 0.0, "correct": 0, "total": 2, "accuracy_percent": 0.0 }, "kobalt:Semantics:Inter-sentence Relations": { "accuracy": 0.0, "correct": 0, "total": 1, "accuracy_percent": 0.0 }, "kobalt:Semantics:Rhetorical Expressions": { "accuracy": 0.3333333333333333, "correct": 1, "total": 3, "accuracy_percent": 33.3333 }, "kobalt:Semantics:Semantic Compatibility": { "accuracy": 0.14285714285714285, "correct": 1, "total": 7, "accuracy_percent": 14.2857 }, "kobalt:Semantics:Word Relationships": { "accuracy": 0.13636363636363635, "correct": 3, "total": 22, "accuracy_percent": 13.6364 }, "kobalt:Syntax:Agreement": { "accuracy": 0.09803921568627451, "correct": 5, "total": 51, "accuracy_percent": 9.8039 }, "kobalt:Syntax:Argument Structure & Valency": { "accuracy": 0.12121212121212122, "correct": 4, "total": 33, "accuracy_percent": 12.1212 }, "kobalt:Syntax:Embedded Clauses": { "accuracy": 0.041666666666666664, "correct": 1, "total": 24, "accuracy_percent": 4.1667 } } }, "kopiqa": { "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/kopiqa.json", "accuracy": 0.49, "accuracy_percent": 49.0, "correct": 98, "total": 200, "score": "avg_logprob", "categories": { "ko-piqa:1.0": { "accuracy": 0.5681818181818182, "correct": 25, "total": 44, "accuracy_percent": 56.8182 }, "ko-piqa:None": { "accuracy": 0.46794871794871795, "correct": 73, "total": 156, "accuracy_percent": 46.7949 } } }, "korean_mini_mcq": { "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/korean_mini_mcq.json", "accuracy": 0.6, "accuracy_percent": 60.0, "correct": 12, "total": 20, "score": "avg_logprob", "categories": { "commonsense": { "accuracy": 0.3333333333333333, "correct": 1, "total": 3, "accuracy_percent": 33.3333 }, "culture": { "accuracy": 1.0, "correct": 1, "total": 1, "accuracy_percent": 100.0 }, "instruction": { "accuracy": 1.0, "correct": 3, "total": 3, "accuracy_percent": 100.0 }, "logic": { "accuracy": 0.6666666666666666, "correct": 2, "total": 3, "accuracy_percent": 66.6667 }, "math": { "accuracy": 0.3333333333333333, "correct": 1, "total": 3, "accuracy_percent": 33.3333 }, "reading": { "accuracy": 0.6666666666666666, "correct": 2, "total": 3, "accuracy_percent": 66.6667 }, "safety": { "accuracy": 0.5, "correct": 1, "total": 2, "accuracy_percent": 50.0 }, "summary": { "accuracy": 0.5, "correct": 1, "total": 2, "accuracy_percent": 50.0 } } }, "winogrande_ko": { "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/winogrande_ko.json", "accuracy": 0.455, "accuracy_percent": 45.5, "correct": 91, "total": 200, "score": "avg_logprob", "categories": { "winogrande-ko": { "accuracy": 0.455, "correct": 91, "total": 200, "accuracy_percent": 45.5 } } } }, "boundary_and_forensic_diagnostics": { "passk32_argmax_boundary_pairs": { "source_json": "/tmp/chojm_hw2_runs/argmax_boundary_eval_20260609/clean7020_passk32/metrics.json", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "settings": { "pairs": 418, "top_k": 16, "margin_threshold": 0.55, "with_rank": true }, "overall": { "count": 418, "target_top1": 0, "target_top1_rate": 0.0, "target_topk": 327, "target_topk_rate": 0.7822966507177034, "target_gt_negative": 0, "target_gt_negative_rate": 0.0, "target_margin_ge_0": 1, "target_margin_ge_0_rate": 0.0023923444976076554, "target_margin_ge_threshold": 0, "target_margin_ge_threshold_rate": 0.0, "margin_mean": -2.341340279465087, "target_logit_mean": 7.655613219338741, "negative_logit_mean": 9.996953498803828, "target_rank_mean": 516.4282296650717 }, "by_task": [ { "task_type": "clean7020_passk32_inventory_accepted_only_instruction", "count": 314, "target_top1": 0, "target_top1_rate": 0.0, "target_topk": 244, "target_topk_rate": 0.7770700636942676, "target_gt_negative": 0, "target_gt_negative_rate": 0.0, "target_margin_ge_0": 1, "target_margin_ge_0_rate": 0.0031847133757961785, "target_margin_ge_threshold": 0, "target_margin_ge_threshold_rate": 0.0, "margin_mean": -2.4404563174885547, "target_logit_mean": 7.31813992056877, "negative_logit_mean": 9.758596238057326, "target_rank_mean": 409.171974522293 }, { "task_type": "clean7020_passk32_inventory_accepted_only_math", "count": 69, "target_top1": 0, "target_top1_rate": 0.0, "target_topk": 60, "target_topk_rate": 0.8695652173913043, "target_gt_negative": 0, "target_gt_negative_rate": 0.0, "target_margin_ge_0": 0, "target_margin_ge_0_rate": 0.0, "target_margin_ge_threshold": 0, "target_margin_ge_threshold_rate": 0.0, "margin_mean": -1.7986016757246377, "target_logit_mean": 9.565755208333334, "negative_logit_mean": 11.36435688405797, "target_rank_mean": 26.942028985507246 }, { "task_type": "clean7020_passk32_inventory_accepted_only_reasoning", "count": 35, "target_top1": 0, "target_top1_rate": 0.0, "target_topk": 23, "target_topk_rate": 0.6571428571428571, "target_gt_negative": 0, "target_gt_negative_rate": 0.0, "target_margin_ge_0": 0, "target_margin_ge_0_rate": 0.0, "target_margin_ge_threshold": 0, "target_margin_ge_threshold_rate": 0.0, "margin_mean": -2.5220982142857142, "target_logit_mean": 6.917522321428572, "negative_logit_mean": 9.439620535714285, "target_rank_mean": 2443.657142857143 } ] }, "gspo_scoreseq_parent_boundary_pairs": { "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_gspo_scoreseq_v1_boundary_parent7020.json", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "settings": { "pairs": 33226, "top_k": 5, "margin_threshold": 0.65, "with_rank": true }, "overall": { "count": 33226, "target_top1": 588, "target_top1_rate": 0.017696984289411907, "target_topk": 7844, "target_topk_rate": 0.23608017817371937, "target_gt_negative": 662, "target_gt_negative_rate": 0.019924155781616805, "target_margin_ge_0": 679, "target_margin_ge_0_rate": 0.020435803286582798, "target_margin_ge_threshold": 70, "target_margin_ge_threshold_rate": 0.002106783843977608, "margin_mean": -4.19784176139393, "target_logit_mean": 4.610714955296313, "negative_logit_mean": 8.808556716690243, "target_rank_mean": 1509.9462168181544 } }, "trainonly_grounded_bridge_forensics_shard00": { "source_json": "/tmp/chojm_hw2_runs/eval/trainonly_grounded_bridge6990_vs_clean7020_20260609/clean7020_shard00/generation-forensics.json", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "summary": { "greedy|plain": { "rows": 128, "bad": 128, "bad_rate": 1.0, "eos_hit": 36, "eos_hit_rate": 0.28125, "semantic_hit": 0, "semantic_hit_rate": 0.0, "avg_forensic_score": 31.9921875, "flags": { "semantic_weak_miss": 128, "max_new_tokens_hit": 92, "token_3gram_repetition": 99, "char_loop": 37, "punctuation_heavy": 38, "low_hangul_alnum": 35, "role_leak": 1 } } } } }, "public_comparison_and_replacement_references": { "benchmark_results_file": "benchmark_results_20260609.json", "replacement_evidence_file": "replacement_evidence_20260609.json", "benchmark_results_summary": { "metric": "Korean semantic pass@33", "prompt_set": "/tmp/chojm_hw2_runs/data/clean7020_passk32_accepted_imr_20260609/sentinel_imr_30x3_90.jsonl", "decoding": { "include_greedy_rollout": true, "samples_per_prompt": 32, "temperature": 0.7, "top_p": 0.95, "top_k": 50, "max_new_tokens": 64 }, "models": { "Jumini-Ko-1.2B-clean7020": { "params": 1260505088, "language_focus": "Korean from-scratch", "summary_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_imr_k32_gpu2/passk_summary.json", "pass_any": "38/90", "pass_any_score": 42.22222222222222, "pass_rollouts": "131/2884", "pass_rollout_score": 4.542302357836339, "greedy_hit": "0/90", "categories": { "instruction": "26/30", "math": "8/30", "reasoning": "4/30" } }, "TinyLlama-1.1B-intermediate-step-1431k-3T": { "repo_id": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T", "params": 1100000000, "language_focus": "English general base", "summary_json": "/tmp/chojm_hw2_runs/hf_baseline_compare_20260609/tinyllama_1p1b_base_k32/summary.json", "pass_any": "31/90", "pass_any_score": 34.44444444444444, "pass_rollouts": "56/2966", "pass_rollout_score": 1.8880647336480108, "greedy_hit": "0/90", "categories": { "instruction": "27/30", "math": "4/30", "reasoning": "0/30" } } } }, "replacement_summary": { "name": "clean7020", "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", "export_dir": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32", "strat240_pass_any": "21/240", "strat240_pass_rollouts": "29/2105", "strat240_mechanical_bad_rollouts": "856/2105", "k32_pass_any": "38/90", "strict_heldout_stable_semantic_hits": "2/12" }, "previous_hf_reference": { "name": "v3a6200", "remote_sha_before_replace": "369600b6f14950e4bf6c7c43457ed3ee0921663c", "checkpoint": "/raid2/chojm/hw2_llm_from_scratch/experiments/jumini-ko-1.2b-phase2-mixture-v3a-retention-continue-5700-to-6200-2gpu12-mb9/step_00006200/model.pt", "strat240_pass_any": "0/240", "strat240_pass_rollouts": "0/2094", "strat240_mechanical_bad_rollouts": "1755/2094" } }, "omission_policy": [ "No row-level prompt text or completions are included in this public score appendix.", "Non-exported descendants, row-edit probes, merge/soup probes, and alternate 7020-lineage diagnostics are not presented as scores for the current HF checkpoint.", "External baseline comparison scores remain in benchmark_results_20260609.json and are not duplicated here except by reference." ] }