1672 lines
54 KiB
JSON
1672 lines
54 KiB
JSON
|
{
|
||
|
|
"created_at": "2026-06-09T10:22:09.271056+00:00",
|
||
|
|
"repo_id": "properly59/Jumini-Ko-1.2B",
|
||
|
|
"model_card_scope": "Completed score summaries for the current public clean7020 checkpoint. Row-level prompts/completions and non-exported descendant checkpoints are intentionally omitted.",
|
||
|
|
"current_public_checkpoint": {
|
||
|
|
"name": "clean7020",
|
||
|
|
"native_checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"export_dir": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32",
|
||
|
|
"source_training_step": 7020,
|
||
|
|
"tokens_seen": 1025474560,
|
||
|
|
"hf_remote_sha_before_this_appendix": "82815c2b339f8674792b8a5910c3eb913aab9e60"
|
||
|
|
},
|
||
|
|
"validation": {
|
||
|
|
"transformers_load_smoke": {
|
||
|
|
"source_json": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32/load_smoke.json",
|
||
|
|
"ok": true,
|
||
|
|
"model_class": "LlamaForCausalLM",
|
||
|
|
"parameter_count": 1260505088,
|
||
|
|
"tokenizer_len": 128000
|
||
|
|
},
|
||
|
|
"hf_export_equivalence": {
|
||
|
|
"source_json": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32/hf_equivalence_atol2e-5.json",
|
||
|
|
"ok": true,
|
||
|
|
"atol": 2e-05,
|
||
|
|
"rtol": 2e-05,
|
||
|
|
"max_abs_diff": 1.4781951904296875e-05,
|
||
|
|
"mean_abs_diff": 1.073120984074194e-06,
|
||
|
|
"rms_diff": 1.5380918512164499e-06,
|
||
|
|
"argmax_mismatch_rate": 0.0
|
||
|
|
},
|
||
|
|
"native_checkpoint_summary": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/argmax_boundary_layernormshock_rerun20260608_gate/step_00007020/checkpoint-summary.json",
|
||
|
|
"ok": true,
|
||
|
|
"step": 7020,
|
||
|
|
"tokens_seen": 1025474560,
|
||
|
|
"file_size_bytes": 6090713562,
|
||
|
|
"native_tensor_parameter_count": 1522649088
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"strict_heldout_project_gate": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/argmax_boundary_layernormshock_rerun20260608_gate/step_00007020/heldout-semantic-gate.json",
|
||
|
|
"preset": "greedy|plain",
|
||
|
|
"rows": 12,
|
||
|
|
"semantic_hits": {
|
||
|
|
"count": 2,
|
||
|
|
"total": 12,
|
||
|
|
"rate": 0.16666666666666666,
|
||
|
|
"percent": 16.6667
|
||
|
|
},
|
||
|
|
"stable_semantic_hits": {
|
||
|
|
"count": 2,
|
||
|
|
"total": 12,
|
||
|
|
"rate": 0.16666666666666666,
|
||
|
|
"percent": 16.6667
|
||
|
|
},
|
||
|
|
"eos_hits": {
|
||
|
|
"count": 5,
|
||
|
|
"total": 12,
|
||
|
|
"rate": 0.4166666666666667,
|
||
|
|
"percent": 41.6667
|
||
|
|
},
|
||
|
|
"max_new_tokens_hits": {
|
||
|
|
"count": 7,
|
||
|
|
"total": 12,
|
||
|
|
"rate": 0.5833333333333334,
|
||
|
|
"percent": 58.3333
|
||
|
|
},
|
||
|
|
"mechanical_bad": {
|
||
|
|
"count": 9,
|
||
|
|
"total": 12,
|
||
|
|
"rate": 0.75,
|
||
|
|
"percent": 75.0
|
||
|
|
},
|
||
|
|
"empty": {
|
||
|
|
"count": 1,
|
||
|
|
"total": 12,
|
||
|
|
"rate": 0.08333333333333333,
|
||
|
|
"percent": 8.3333
|
||
|
|
},
|
||
|
|
"semantic_hit_ids": [
|
||
|
|
"heldout_summary_battery_001",
|
||
|
|
"heldout_minutes_split_001"
|
||
|
|
],
|
||
|
|
"flag_counts": {
|
||
|
|
"max_new_tokens_hit": 7,
|
||
|
|
"token_3gram_repetition": 8,
|
||
|
|
"punctuation_heavy": 3,
|
||
|
|
"char_loop": 3,
|
||
|
|
"low_hangul_alnum": 3,
|
||
|
|
"empty": 1
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"korean_semantic_passk_and_decode_diagnostics": {
|
||
|
|
"k32_korean_semantic_sentinel_90": {
|
||
|
|
"label": "90-prompt Korean semantic sentinel, greedy + 32 sampled rollouts",
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_imr_k32_gpu2/passk_summary.json",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"settings": {
|
||
|
|
"eos_logit_bias": 0.0,
|
||
|
|
"include_greedy_rollout": true,
|
||
|
|
"max_new_tokens": 64,
|
||
|
|
"no_repeat_ngram_size": 0,
|
||
|
|
"precision": "fp16",
|
||
|
|
"prompt_variant": "plain",
|
||
|
|
"repetition_penalty": 1.0,
|
||
|
|
"samples_per_prompt": 32,
|
||
|
|
"seed": 20260609,
|
||
|
|
"temperature": 0.7,
|
||
|
|
"top_k": 50,
|
||
|
|
"top_p": 0.95
|
||
|
|
},
|
||
|
|
"totals": {
|
||
|
|
"evaluated_prompts": 90,
|
||
|
|
"read_prompts": 90,
|
||
|
|
"rollouts": 2884,
|
||
|
|
"pass_any_prompts": 38,
|
||
|
|
"pass_any_rate": 0.4222222222222222,
|
||
|
|
"pass_rollouts": 131,
|
||
|
|
"pass_rollout_rate": 0.045423023578363385,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 38,
|
||
|
|
"sample_hit_rate": 0.4222222222222222,
|
||
|
|
"eos_hits": 2071,
|
||
|
|
"eos_hit_rate": 0.7180998613037448,
|
||
|
|
"mechanical_bad_rollouts": 930,
|
||
|
|
"mechanical_bad_rate": 0.3224687933425798,
|
||
|
|
"pass_any_categories": [
|
||
|
|
"instruction",
|
||
|
|
"math",
|
||
|
|
"reasoning"
|
||
|
|
],
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 222,
|
||
|
|
"punctuation_heavy": 226,
|
||
|
|
"role_leak": 8,
|
||
|
|
"token_3gram_repetition": 914
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"by_category": {
|
||
|
|
"instruction": {
|
||
|
|
"rollouts": 913,
|
||
|
|
"pass_any_prompts": 26,
|
||
|
|
"pass_any_rate": 0.8666666666666667,
|
||
|
|
"pass_rollouts": 110,
|
||
|
|
"pass_rollout_rate": 0.12048192771084337,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 26,
|
||
|
|
"sample_hit_rate": 0.8666666666666667,
|
||
|
|
"eos_hits": 561,
|
||
|
|
"eos_hit_rate": 0.6144578313253012,
|
||
|
|
"mechanical_bad_rollouts": 384,
|
||
|
|
"mechanical_bad_rate": 0.42059145673603504,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 152,
|
||
|
|
"punctuation_heavy": 161,
|
||
|
|
"token_3gram_repetition": 373
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"math": {
|
||
|
|
"rollouts": 986,
|
||
|
|
"pass_any_prompts": 8,
|
||
|
|
"pass_any_rate": 0.26666666666666666,
|
||
|
|
"pass_rollouts": 14,
|
||
|
|
"pass_rollout_rate": 0.014198782961460446,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 8,
|
||
|
|
"sample_hit_rate": 0.26666666666666666,
|
||
|
|
"eos_hits": 743,
|
||
|
|
"eos_hit_rate": 0.7535496957403651,
|
||
|
|
"mechanical_bad_rollouts": 311,
|
||
|
|
"mechanical_bad_rate": 0.3154158215010142,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 18,
|
||
|
|
"punctuation_heavy": 15,
|
||
|
|
"role_leak": 8,
|
||
|
|
"token_3gram_repetition": 306
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"reasoning": {
|
||
|
|
"rollouts": 985,
|
||
|
|
"pass_any_prompts": 4,
|
||
|
|
"pass_any_rate": 0.13333333333333333,
|
||
|
|
"pass_rollouts": 7,
|
||
|
|
"pass_rollout_rate": 0.007106598984771574,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 4,
|
||
|
|
"sample_hit_rate": 0.13333333333333333,
|
||
|
|
"eos_hits": 767,
|
||
|
|
"eos_hit_rate": 0.7786802030456853,
|
||
|
|
"mechanical_bad_rollouts": 235,
|
||
|
|
"mechanical_bad_rate": 0.23857868020304568,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 52,
|
||
|
|
"punctuation_heavy": 50,
|
||
|
|
"token_3gram_repetition": 235
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"stratified_korean_semantic_sentinel_240": {
|
||
|
|
"label": "240-prompt stratified Korean semantic sentinel",
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_strat240_gpu2/passk_summary.json",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"settings": {
|
||
|
|
"eos_logit_bias": 0.0,
|
||
|
|
"include_greedy_rollout": true,
|
||
|
|
"max_new_tokens": 64,
|
||
|
|
"no_repeat_ngram_size": 0,
|
||
|
|
"precision": "fp16",
|
||
|
|
"prompt_variant": "plain",
|
||
|
|
"repetition_penalty": 1.0,
|
||
|
|
"samples_per_prompt": 8,
|
||
|
|
"seed": 20260609,
|
||
|
|
"temperature": 0.7,
|
||
|
|
"top_k": 50,
|
||
|
|
"top_p": 0.95
|
||
|
|
},
|
||
|
|
"totals": {
|
||
|
|
"evaluated_prompts": 240,
|
||
|
|
"read_prompts": 240,
|
||
|
|
"rollouts": 2105,
|
||
|
|
"pass_any_prompts": 21,
|
||
|
|
"pass_any_rate": 0.0875,
|
||
|
|
"pass_rollouts": 29,
|
||
|
|
"pass_rollout_rate": 0.013776722090261283,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 21,
|
||
|
|
"sample_hit_rate": 0.0875,
|
||
|
|
"eos_hits": 1238,
|
||
|
|
"eos_hit_rate": 0.58812351543943,
|
||
|
|
"mechanical_bad_rollouts": 856,
|
||
|
|
"mechanical_bad_rate": 0.4066508313539192,
|
||
|
|
"pass_any_categories": [
|
||
|
|
"instruction",
|
||
|
|
"math",
|
||
|
|
"reasoning"
|
||
|
|
],
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 230,
|
||
|
|
"punctuation_heavy": 214,
|
||
|
|
"role_leak": 17,
|
||
|
|
"token_3gram_repetition": 838
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"by_category": {
|
||
|
|
"causal": {
|
||
|
|
"rollouts": 262,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 122,
|
||
|
|
"eos_hit_rate": 0.46564885496183206,
|
||
|
|
"mechanical_bad_rollouts": 156,
|
||
|
|
"mechanical_bad_rate": 0.5954198473282443,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 37,
|
||
|
|
"punctuation_heavy": 35,
|
||
|
|
"token_3gram_repetition": 156
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"compare": {
|
||
|
|
"rollouts": 265,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 155,
|
||
|
|
"eos_hit_rate": 0.5849056603773585,
|
||
|
|
"mechanical_bad_rollouts": 124,
|
||
|
|
"mechanical_bad_rate": 0.4679245283018868,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 25,
|
||
|
|
"punctuation_heavy": 25,
|
||
|
|
"role_leak": 3,
|
||
|
|
"token_3gram_repetition": 122
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"instruction": {
|
||
|
|
"rollouts": 255,
|
||
|
|
"pass_any_prompts": 17,
|
||
|
|
"pass_any_rate": 0.5666666666666667,
|
||
|
|
"pass_rollouts": 24,
|
||
|
|
"pass_rollout_rate": 0.09411764705882353,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 17,
|
||
|
|
"sample_hit_rate": 0.5666666666666667,
|
||
|
|
"eos_hits": 149,
|
||
|
|
"eos_hit_rate": 0.5843137254901961,
|
||
|
|
"mechanical_bad_rollouts": 119,
|
||
|
|
"mechanical_bad_rate": 0.4666666666666667,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 64,
|
||
|
|
"punctuation_heavy": 66,
|
||
|
|
"token_3gram_repetition": 116
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"math": {
|
||
|
|
"rollouts": 270,
|
||
|
|
"pass_any_prompts": 2,
|
||
|
|
"pass_any_rate": 0.06666666666666667,
|
||
|
|
"pass_rollouts": 2,
|
||
|
|
"pass_rollout_rate": 0.007407407407407408,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 2,
|
||
|
|
"sample_hit_rate": 0.06666666666666667,
|
||
|
|
"eos_hits": 207,
|
||
|
|
"eos_hit_rate": 0.7666666666666667,
|
||
|
|
"mechanical_bad_rollouts": 92,
|
||
|
|
"mechanical_bad_rate": 0.34074074074074073,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 4,
|
||
|
|
"punctuation_heavy": 3,
|
||
|
|
"role_leak": 4,
|
||
|
|
"token_3gram_repetition": 90
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"notice": {
|
||
|
|
"rollouts": 270,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 71,
|
||
|
|
"eos_hit_rate": 0.26296296296296295,
|
||
|
|
"mechanical_bad_rollouts": 99,
|
||
|
|
"mechanical_bad_rate": 0.36666666666666664,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 19,
|
||
|
|
"punctuation_heavy": 8,
|
||
|
|
"token_3gram_repetition": 96
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"reasoning": {
|
||
|
|
"rollouts": 269,
|
||
|
|
"pass_any_prompts": 2,
|
||
|
|
"pass_any_rate": 0.06666666666666667,
|
||
|
|
"pass_rollouts": 3,
|
||
|
|
"pass_rollout_rate": 0.011152416356877323,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 2,
|
||
|
|
"sample_hit_rate": 0.06666666666666667,
|
||
|
|
"eos_hits": 207,
|
||
|
|
"eos_hit_rate": 0.7695167286245354,
|
||
|
|
"mechanical_bad_rollouts": 59,
|
||
|
|
"mechanical_bad_rate": 0.21933085501858737,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 7,
|
||
|
|
"punctuation_heavy": 9,
|
||
|
|
"token_3gram_repetition": 59
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"safety": {
|
||
|
|
"rollouts": 258,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 156,
|
||
|
|
"eos_hit_rate": 0.6046511627906976,
|
||
|
|
"mechanical_bad_rollouts": 111,
|
||
|
|
"mechanical_bad_rate": 0.43023255813953487,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 30,
|
||
|
|
"punctuation_heavy": 29,
|
||
|
|
"role_leak": 9,
|
||
|
|
"token_3gram_repetition": 104
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"style": {
|
||
|
|
"rollouts": 256,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 171,
|
||
|
|
"eos_hit_rate": 0.66796875,
|
||
|
|
"mechanical_bad_rollouts": 96,
|
||
|
|
"mechanical_bad_rate": 0.375,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 44,
|
||
|
|
"punctuation_heavy": 39,
|
||
|
|
"role_leak": 1,
|
||
|
|
"token_3gram_repetition": 95
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"k3_plain64_baseline_90": {
|
||
|
|
"label": "90-prompt Korean semantic sentinel, greedy + 3 sampled rollouts",
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/checkpoint_interpolation_20260609/clean7020_baseline/passk_imr_k3_plain64/passk_summary.json",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"settings": {
|
||
|
|
"eos_logit_bias": 0.0,
|
||
|
|
"include_greedy_rollout": true,
|
||
|
|
"max_new_tokens": 64,
|
||
|
|
"no_repeat_ngram_size": 0,
|
||
|
|
"precision": "fp16",
|
||
|
|
"prompt_variant": "plain",
|
||
|
|
"repetition_penalty": 1.0,
|
||
|
|
"samples_per_prompt": 3,
|
||
|
|
"seed": 20260609,
|
||
|
|
"temperature": 0.7,
|
||
|
|
"top_k": 50,
|
||
|
|
"top_p": 0.95
|
||
|
|
},
|
||
|
|
"totals": {
|
||
|
|
"evaluated_prompts": 90,
|
||
|
|
"read_prompts": 90,
|
||
|
|
"rollouts": 357,
|
||
|
|
"pass_any_prompts": 11,
|
||
|
|
"pass_any_rate": 0.12222222222222222,
|
||
|
|
"pass_rollouts": 11,
|
||
|
|
"pass_rollout_rate": 0.03081232492997199,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 11,
|
||
|
|
"sample_hit_rate": 0.12222222222222222,
|
||
|
|
"eos_hits": 224,
|
||
|
|
"eos_hit_rate": 0.6274509803921569,
|
||
|
|
"mechanical_bad_rollouts": 163,
|
||
|
|
"mechanical_bad_rate": 0.4565826330532213,
|
||
|
|
"pass_any_categories": [
|
||
|
|
"instruction",
|
||
|
|
"math"
|
||
|
|
],
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 56,
|
||
|
|
"punctuation_heavy": 56,
|
||
|
|
"token_3gram_repetition": 161
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"by_category": {
|
||
|
|
"instruction": {
|
||
|
|
"rollouts": 119,
|
||
|
|
"pass_any_prompts": 10,
|
||
|
|
"pass_any_rate": 0.3333333333333333,
|
||
|
|
"pass_rollouts": 10,
|
||
|
|
"pass_rollout_rate": 0.08403361344537816,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 10,
|
||
|
|
"sample_hit_rate": 0.3333333333333333,
|
||
|
|
"eos_hits": 58,
|
||
|
|
"eos_hit_rate": 0.48739495798319327,
|
||
|
|
"mechanical_bad_rollouts": 67,
|
||
|
|
"mechanical_bad_rate": 0.5630252100840336,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 44,
|
||
|
|
"punctuation_heavy": 45,
|
||
|
|
"token_3gram_repetition": 65
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"math": {
|
||
|
|
"rollouts": 119,
|
||
|
|
"pass_any_prompts": 1,
|
||
|
|
"pass_any_rate": 0.03333333333333333,
|
||
|
|
"pass_rollouts": 1,
|
||
|
|
"pass_rollout_rate": 0.008403361344537815,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 1,
|
||
|
|
"sample_hit_rate": 0.03333333333333333,
|
||
|
|
"eos_hits": 73,
|
||
|
|
"eos_hit_rate": 0.6134453781512605,
|
||
|
|
"mechanical_bad_rollouts": 61,
|
||
|
|
"mechanical_bad_rate": 0.5126050420168067,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 5,
|
||
|
|
"punctuation_heavy": 3,
|
||
|
|
"token_3gram_repetition": 61
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"reasoning": {
|
||
|
|
"rollouts": 119,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 93,
|
||
|
|
"eos_hit_rate": 0.7815126050420168,
|
||
|
|
"mechanical_bad_rollouts": 35,
|
||
|
|
"mechanical_bad_rate": 0.29411764705882354,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 7,
|
||
|
|
"punctuation_heavy": 8,
|
||
|
|
"token_3gram_repetition": 35
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"greedy_only_90": {
|
||
|
|
"label": "90-prompt Korean semantic sentinel, greedy only",
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_sentinel_imr_greedy_gpu1/passk_summary.json",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"settings": {
|
||
|
|
"eos_logit_bias": 0.0,
|
||
|
|
"include_greedy_rollout": true,
|
||
|
|
"max_new_tokens": 64,
|
||
|
|
"no_repeat_ngram_size": 0,
|
||
|
|
"precision": "fp16",
|
||
|
|
"prompt_variant": "plain",
|
||
|
|
"repetition_penalty": 1.0,
|
||
|
|
"samples_per_prompt": 0,
|
||
|
|
"seed": 20260610,
|
||
|
|
"temperature": 0.7,
|
||
|
|
"top_k": 50,
|
||
|
|
"top_p": 0.95
|
||
|
|
},
|
||
|
|
"totals": {
|
||
|
|
"evaluated_prompts": 90,
|
||
|
|
"read_prompts": 90,
|
||
|
|
"rollouts": 90,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 46,
|
||
|
|
"eos_hit_rate": 0.5111111111111111,
|
||
|
|
"mechanical_bad_rollouts": 65,
|
||
|
|
"mechanical_bad_rate": 0.7222222222222222,
|
||
|
|
"pass_any_categories": [],
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 24,
|
||
|
|
"punctuation_heavy": 24,
|
||
|
|
"token_3gram_repetition": 65
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"by_category": {
|
||
|
|
"instruction": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 8,
|
||
|
|
"eos_hit_rate": 0.26666666666666666,
|
||
|
|
"mechanical_bad_rollouts": 22,
|
||
|
|
"mechanical_bad_rate": 0.7333333333333333,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 22,
|
||
|
|
"punctuation_heavy": 22,
|
||
|
|
"token_3gram_repetition": 22
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"math": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 15,
|
||
|
|
"eos_hit_rate": 0.5,
|
||
|
|
"mechanical_bad_rollouts": 27,
|
||
|
|
"mechanical_bad_rate": 0.9,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 1,
|
||
|
|
"punctuation_heavy": 1,
|
||
|
|
"token_3gram_repetition": 27
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"reasoning": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 23,
|
||
|
|
"eos_hit_rate": 0.7666666666666667,
|
||
|
|
"mechanical_bad_rollouts": 16,
|
||
|
|
"mechanical_bad_rate": 0.5333333333333333,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 1,
|
||
|
|
"punctuation_heavy": 1,
|
||
|
|
"token_3gram_repetition": 16
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"two_prompt_smoke": {
|
||
|
|
"label": "2-prompt generation smoke only",
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/smoke_clean7020/summary.json",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"settings": {
|
||
|
|
"eos_logit_bias": 0.0,
|
||
|
|
"include_greedy_rollout": true,
|
||
|
|
"max_new_tokens": 16,
|
||
|
|
"no_repeat_ngram_size": 0,
|
||
|
|
"precision": "fp16",
|
||
|
|
"prompt_variant": "plain",
|
||
|
|
"repetition_penalty": 1.0,
|
||
|
|
"samples_per_prompt": 1,
|
||
|
|
"seed": 20260609,
|
||
|
|
"temperature": 0.7,
|
||
|
|
"top_k": 50,
|
||
|
|
"top_p": 0.95
|
||
|
|
},
|
||
|
|
"totals": {
|
||
|
|
"evaluated_prompts": 2,
|
||
|
|
"read_prompts": 2,
|
||
|
|
"rollouts": 4,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 0,
|
||
|
|
"eos_hit_rate": 0.0,
|
||
|
|
"mechanical_bad_rollouts": 1,
|
||
|
|
"mechanical_bad_rate": 0.25,
|
||
|
|
"pass_any_categories": [],
|
||
|
|
"mechanical_flags": {
|
||
|
|
"token_3gram_repetition": 1
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"by_category": {
|
||
|
|
"compare": {
|
||
|
|
"rollouts": 2,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 0,
|
||
|
|
"eos_hit_rate": 0.0,
|
||
|
|
"mechanical_bad_rollouts": 1,
|
||
|
|
"mechanical_bad_rate": 0.5,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"token_3gram_repetition": 1
|
||
|
|
},
|
||
|
|
"prompts": 1
|
||
|
|
},
|
||
|
|
"notice": {
|
||
|
|
"rollouts": 2,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 0,
|
||
|
|
"eos_hit_rate": 0.0,
|
||
|
|
"mechanical_bad_rollouts": 0,
|
||
|
|
"mechanical_bad_rate": 0.0,
|
||
|
|
"mechanical_flags": {},
|
||
|
|
"prompts": 1
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"decode_control_eosbias1": {
|
||
|
|
"label": "90-prompt greedy decode control: eosbias1",
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_decode_sentinel/eosbias1/passk_summary.json",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"settings": {
|
||
|
|
"eos_logit_bias": 1.0,
|
||
|
|
"include_greedy_rollout": true,
|
||
|
|
"max_new_tokens": 64,
|
||
|
|
"no_repeat_ngram_size": 0,
|
||
|
|
"precision": "fp16",
|
||
|
|
"prompt_variant": "plain",
|
||
|
|
"repetition_penalty": 1.0,
|
||
|
|
"samples_per_prompt": 0,
|
||
|
|
"seed": 20260610,
|
||
|
|
"temperature": 0.7,
|
||
|
|
"top_k": 50,
|
||
|
|
"top_p": 0.95
|
||
|
|
},
|
||
|
|
"totals": {
|
||
|
|
"evaluated_prompts": 90,
|
||
|
|
"read_prompts": 90,
|
||
|
|
"rollouts": 90,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 46,
|
||
|
|
"eos_hit_rate": 0.5111111111111111,
|
||
|
|
"mechanical_bad_rollouts": 65,
|
||
|
|
"mechanical_bad_rate": 0.7222222222222222,
|
||
|
|
"pass_any_categories": [],
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 24,
|
||
|
|
"punctuation_heavy": 24,
|
||
|
|
"token_3gram_repetition": 65
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"by_category": {
|
||
|
|
"instruction": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 8,
|
||
|
|
"eos_hit_rate": 0.26666666666666666,
|
||
|
|
"mechanical_bad_rollouts": 22,
|
||
|
|
"mechanical_bad_rate": 0.7333333333333333,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 22,
|
||
|
|
"punctuation_heavy": 22,
|
||
|
|
"token_3gram_repetition": 22
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"math": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 15,
|
||
|
|
"eos_hit_rate": 0.5,
|
||
|
|
"mechanical_bad_rollouts": 27,
|
||
|
|
"mechanical_bad_rate": 0.9,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 1,
|
||
|
|
"punctuation_heavy": 1,
|
||
|
|
"token_3gram_repetition": 27
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"reasoning": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 23,
|
||
|
|
"eos_hit_rate": 0.7666666666666667,
|
||
|
|
"mechanical_bad_rollouts": 16,
|
||
|
|
"mechanical_bad_rate": 0.5333333333333333,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 1,
|
||
|
|
"punctuation_heavy": 1,
|
||
|
|
"token_3gram_repetition": 16
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"decode_control_no_repeat3": {
|
||
|
|
"label": "90-prompt greedy decode control: no_repeat3",
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_decode_sentinel/no_repeat3/passk_summary.json",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"settings": {
|
||
|
|
"eos_logit_bias": 0.0,
|
||
|
|
"include_greedy_rollout": true,
|
||
|
|
"max_new_tokens": 64,
|
||
|
|
"no_repeat_ngram_size": 3,
|
||
|
|
"precision": "fp16",
|
||
|
|
"prompt_variant": "plain",
|
||
|
|
"repetition_penalty": 1.0,
|
||
|
|
"samples_per_prompt": 0,
|
||
|
|
"seed": 20260610,
|
||
|
|
"temperature": 0.7,
|
||
|
|
"top_k": 50,
|
||
|
|
"top_p": 0.95
|
||
|
|
},
|
||
|
|
"totals": {
|
||
|
|
"evaluated_prompts": 90,
|
||
|
|
"read_prompts": 90,
|
||
|
|
"rollouts": 90,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 46,
|
||
|
|
"eos_hit_rate": 0.5111111111111111,
|
||
|
|
"mechanical_bad_rollouts": 65,
|
||
|
|
"mechanical_bad_rate": 0.7222222222222222,
|
||
|
|
"pass_any_categories": [],
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 24,
|
||
|
|
"punctuation_heavy": 24,
|
||
|
|
"token_3gram_repetition": 65
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"by_category": {
|
||
|
|
"instruction": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 8,
|
||
|
|
"eos_hit_rate": 0.26666666666666666,
|
||
|
|
"mechanical_bad_rollouts": 22,
|
||
|
|
"mechanical_bad_rate": 0.7333333333333333,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 22,
|
||
|
|
"punctuation_heavy": 22,
|
||
|
|
"token_3gram_repetition": 22
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"math": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 15,
|
||
|
|
"eos_hit_rate": 0.5,
|
||
|
|
"mechanical_bad_rollouts": 27,
|
||
|
|
"mechanical_bad_rate": 0.9,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 1,
|
||
|
|
"punctuation_heavy": 1,
|
||
|
|
"token_3gram_repetition": 27
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"reasoning": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 23,
|
||
|
|
"eos_hit_rate": 0.7666666666666667,
|
||
|
|
"mechanical_bad_rollouts": 16,
|
||
|
|
"mechanical_bad_rate": 0.5333333333333333,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 1,
|
||
|
|
"punctuation_heavy": 1,
|
||
|
|
"token_3gram_repetition": 16
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"decode_control_rep108": {
|
||
|
|
"label": "90-prompt greedy decode control: rep108",
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_decode_sentinel/rep108/passk_summary.json",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"settings": {
|
||
|
|
"eos_logit_bias": 0.0,
|
||
|
|
"include_greedy_rollout": true,
|
||
|
|
"max_new_tokens": 64,
|
||
|
|
"no_repeat_ngram_size": 0,
|
||
|
|
"precision": "fp16",
|
||
|
|
"prompt_variant": "plain",
|
||
|
|
"repetition_penalty": 1.08,
|
||
|
|
"samples_per_prompt": 0,
|
||
|
|
"seed": 20260610,
|
||
|
|
"temperature": 0.7,
|
||
|
|
"top_k": 50,
|
||
|
|
"top_p": 0.95
|
||
|
|
},
|
||
|
|
"totals": {
|
||
|
|
"evaluated_prompts": 90,
|
||
|
|
"read_prompts": 90,
|
||
|
|
"rollouts": 90,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 46,
|
||
|
|
"eos_hit_rate": 0.5111111111111111,
|
||
|
|
"mechanical_bad_rollouts": 65,
|
||
|
|
"mechanical_bad_rate": 0.7222222222222222,
|
||
|
|
"pass_any_categories": [],
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 24,
|
||
|
|
"punctuation_heavy": 24,
|
||
|
|
"token_3gram_repetition": 65
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"by_category": {
|
||
|
|
"instruction": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 8,
|
||
|
|
"eos_hit_rate": 0.26666666666666666,
|
||
|
|
"mechanical_bad_rollouts": 22,
|
||
|
|
"mechanical_bad_rate": 0.7333333333333333,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 22,
|
||
|
|
"punctuation_heavy": 22,
|
||
|
|
"token_3gram_repetition": 22
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"math": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 15,
|
||
|
|
"eos_hit_rate": 0.5,
|
||
|
|
"mechanical_bad_rollouts": 27,
|
||
|
|
"mechanical_bad_rate": 0.9,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 1,
|
||
|
|
"punctuation_heavy": 1,
|
||
|
|
"token_3gram_repetition": 27
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"reasoning": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 23,
|
||
|
|
"eos_hit_rate": 0.7666666666666667,
|
||
|
|
"mechanical_bad_rollouts": 16,
|
||
|
|
"mechanical_bad_rate": 0.5333333333333333,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 1,
|
||
|
|
"punctuation_heavy": 1,
|
||
|
|
"token_3gram_repetition": 16
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"decode_control_rep108_eos1": {
|
||
|
|
"label": "90-prompt greedy decode control: rep108_eos1",
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_decode_sentinel/rep108_eos1/passk_summary.json",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"settings": {
|
||
|
|
"eos_logit_bias": 1.0,
|
||
|
|
"include_greedy_rollout": true,
|
||
|
|
"max_new_tokens": 64,
|
||
|
|
"no_repeat_ngram_size": 0,
|
||
|
|
"precision": "fp16",
|
||
|
|
"prompt_variant": "plain",
|
||
|
|
"repetition_penalty": 1.08,
|
||
|
|
"samples_per_prompt": 0,
|
||
|
|
"seed": 20260610,
|
||
|
|
"temperature": 0.7,
|
||
|
|
"top_k": 50,
|
||
|
|
"top_p": 0.95
|
||
|
|
},
|
||
|
|
"totals": {
|
||
|
|
"evaluated_prompts": 90,
|
||
|
|
"read_prompts": 90,
|
||
|
|
"rollouts": 90,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 46,
|
||
|
|
"eos_hit_rate": 0.5111111111111111,
|
||
|
|
"mechanical_bad_rollouts": 65,
|
||
|
|
"mechanical_bad_rate": 0.7222222222222222,
|
||
|
|
"pass_any_categories": [],
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 24,
|
||
|
|
"punctuation_heavy": 24,
|
||
|
|
"token_3gram_repetition": 65
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"by_category": {
|
||
|
|
"instruction": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 8,
|
||
|
|
"eos_hit_rate": 0.26666666666666666,
|
||
|
|
"mechanical_bad_rollouts": 22,
|
||
|
|
"mechanical_bad_rate": 0.7333333333333333,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 22,
|
||
|
|
"punctuation_heavy": 22,
|
||
|
|
"token_3gram_repetition": 22
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"math": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 15,
|
||
|
|
"eos_hit_rate": 0.5,
|
||
|
|
"mechanical_bad_rollouts": 27,
|
||
|
|
"mechanical_bad_rate": 0.9,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 1,
|
||
|
|
"punctuation_heavy": 1,
|
||
|
|
"token_3gram_repetition": 27
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
},
|
||
|
|
"reasoning": {
|
||
|
|
"rollouts": 30,
|
||
|
|
"pass_any_prompts": 0,
|
||
|
|
"pass_any_rate": 0.0,
|
||
|
|
"pass_rollouts": 0,
|
||
|
|
"pass_rollout_rate": 0.0,
|
||
|
|
"greedy_hit_prompts": 0,
|
||
|
|
"greedy_hit_rate": 0.0,
|
||
|
|
"sample_hit_prompts": 0,
|
||
|
|
"sample_hit_rate": 0.0,
|
||
|
|
"eos_hits": 23,
|
||
|
|
"eos_hit_rate": 0.7666666666666667,
|
||
|
|
"mechanical_bad_rollouts": 16,
|
||
|
|
"mechanical_bad_rate": 0.5333333333333333,
|
||
|
|
"mechanical_flags": {
|
||
|
|
"char_loop": 1,
|
||
|
|
"punctuation_heavy": 1,
|
||
|
|
"token_3gram_repetition": 16
|
||
|
|
},
|
||
|
|
"prompts": 30
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"korean_public_proxy_stagegate": {
|
||
|
|
"belebele_kor": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/belebele_kor.json",
|
||
|
|
"accuracy": 0.275,
|
||
|
|
"accuracy_percent": 27.5,
|
||
|
|
"correct": 55,
|
||
|
|
"total": 200,
|
||
|
|
"score": "avg_logprob",
|
||
|
|
"categories": {
|
||
|
|
"belebele:kor_Hang": {
|
||
|
|
"accuracy": 0.275,
|
||
|
|
"correct": 55,
|
||
|
|
"total": 200,
|
||
|
|
"accuracy_percent": 27.5
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"gsm8k_ko": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/gsm8k_ko.json",
|
||
|
|
"accuracy": 0.0,
|
||
|
|
"accuracy_percent": 0.0,
|
||
|
|
"correct": 0,
|
||
|
|
"total": 100,
|
||
|
|
"max_new_tokens": 192
|
||
|
|
},
|
||
|
|
"haerae_bench_2_0": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/haerae_bench_2_0.json",
|
||
|
|
"accuracy": 0.16666666666666666,
|
||
|
|
"accuracy_percent": 16.6667,
|
||
|
|
"correct": 10,
|
||
|
|
"total": 60,
|
||
|
|
"score": "avg_logprob",
|
||
|
|
"categories": {
|
||
|
|
"haerae-bench-2.0:context_definition_alignment": {
|
||
|
|
"accuracy": 0.26666666666666666,
|
||
|
|
"correct": 8,
|
||
|
|
"total": 30,
|
||
|
|
"accuracy_percent": 26.6667
|
||
|
|
},
|
||
|
|
"haerae-bench-2.0:date_understanding": {
|
||
|
|
"accuracy": 0.06666666666666667,
|
||
|
|
"correct": 2,
|
||
|
|
"total": 30,
|
||
|
|
"accuracy_percent": 6.6667
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"hrm8k": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/hrm8k.json",
|
||
|
|
"accuracy": 0.03333333333333333,
|
||
|
|
"accuracy_percent": 3.3333,
|
||
|
|
"correct": 2,
|
||
|
|
"total": 60,
|
||
|
|
"max_new_tokens": 192
|
||
|
|
},
|
||
|
|
"kmmlu_hard": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/kmmlu_hard.json",
|
||
|
|
"accuracy": 0.2125,
|
||
|
|
"accuracy_percent": 21.25,
|
||
|
|
"correct": 34,
|
||
|
|
"total": 160,
|
||
|
|
"score": "avg_logprob",
|
||
|
|
"categories": {
|
||
|
|
"Biology": {
|
||
|
|
"accuracy": 0.2,
|
||
|
|
"correct": 4,
|
||
|
|
"total": 20,
|
||
|
|
"accuracy_percent": 20.0
|
||
|
|
},
|
||
|
|
"Computer Science": {
|
||
|
|
"accuracy": 0.25,
|
||
|
|
"correct": 5,
|
||
|
|
"total": 20,
|
||
|
|
"accuracy_percent": 25.0
|
||
|
|
},
|
||
|
|
"Education": {
|
||
|
|
"accuracy": 0.1,
|
||
|
|
"correct": 2,
|
||
|
|
"total": 20,
|
||
|
|
"accuracy_percent": 10.0
|
||
|
|
},
|
||
|
|
"Environmental Science": {
|
||
|
|
"accuracy": 0.2,
|
||
|
|
"correct": 4,
|
||
|
|
"total": 20,
|
||
|
|
"accuracy_percent": 20.0
|
||
|
|
},
|
||
|
|
"Fashion": {
|
||
|
|
"accuracy": 0.2,
|
||
|
|
"correct": 4,
|
||
|
|
"total": 20,
|
||
|
|
"accuracy_percent": 20.0
|
||
|
|
},
|
||
|
|
"Korean History": {
|
||
|
|
"accuracy": 0.3,
|
||
|
|
"correct": 6,
|
||
|
|
"total": 20,
|
||
|
|
"accuracy_percent": 30.0
|
||
|
|
},
|
||
|
|
"Psychology": {
|
||
|
|
"accuracy": 0.2,
|
||
|
|
"correct": 4,
|
||
|
|
"total": 20,
|
||
|
|
"accuracy_percent": 20.0
|
||
|
|
},
|
||
|
|
"Railway and Automotive Engineering": {
|
||
|
|
"accuracy": 0.25,
|
||
|
|
"correct": 5,
|
||
|
|
"total": 20,
|
||
|
|
"accuracy_percent": 25.0
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"kobalt": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/kobalt.json",
|
||
|
|
"accuracy": 0.095,
|
||
|
|
"accuracy_percent": 9.5,
|
||
|
|
"correct": 19,
|
||
|
|
"total": 200,
|
||
|
|
"score": "avg_logprob",
|
||
|
|
"categories": {
|
||
|
|
"kobalt:Phonetics/Phonology:Phonological Alternation": {
|
||
|
|
"accuracy": 0.1111111111111111,
|
||
|
|
"correct": 1,
|
||
|
|
"total": 9,
|
||
|
|
"accuracy_percent": 11.1111
|
||
|
|
},
|
||
|
|
"kobalt:Phonetics/Phonology:Phonological Constraints": {
|
||
|
|
"accuracy": 0.0,
|
||
|
|
"correct": 0,
|
||
|
|
"total": 3,
|
||
|
|
"accuracy_percent": 0.0
|
||
|
|
},
|
||
|
|
"kobalt:Pragmatics:Deixis & Reference": {
|
||
|
|
"accuracy": 0.0,
|
||
|
|
"correct": 0,
|
||
|
|
"total": 12,
|
||
|
|
"accuracy_percent": 0.0
|
||
|
|
},
|
||
|
|
"kobalt:Pragmatics:Discourse Principles": {
|
||
|
|
"accuracy": 0.0,
|
||
|
|
"correct": 0,
|
||
|
|
"total": 6,
|
||
|
|
"accuracy_percent": 0.0
|
||
|
|
},
|
||
|
|
"kobalt:Pragmatics:Implicature": {
|
||
|
|
"accuracy": 0.14285714285714285,
|
||
|
|
"correct": 1,
|
||
|
|
"total": 7,
|
||
|
|
"accuracy_percent": 14.2857
|
||
|
|
},
|
||
|
|
"kobalt:Pragmatics:Social Relationship Marking": {
|
||
|
|
"accuracy": 0.5,
|
||
|
|
"correct": 1,
|
||
|
|
"total": 2,
|
||
|
|
"accuracy_percent": 50.0
|
||
|
|
},
|
||
|
|
"kobalt:Pragmatics:Speech Acts": {
|
||
|
|
"accuracy": 0.05555555555555555,
|
||
|
|
"correct": 1,
|
||
|
|
"total": 18,
|
||
|
|
"accuracy_percent": 5.5556
|
||
|
|
},
|
||
|
|
"kobalt:Semantics:Conjunctions": {
|
||
|
|
"accuracy": 0.0,
|
||
|
|
"correct": 0,
|
||
|
|
"total": 2,
|
||
|
|
"accuracy_percent": 0.0
|
||
|
|
},
|
||
|
|
"kobalt:Semantics:Inter-sentence Relations": {
|
||
|
|
"accuracy": 0.0,
|
||
|
|
"correct": 0,
|
||
|
|
"total": 1,
|
||
|
|
"accuracy_percent": 0.0
|
||
|
|
},
|
||
|
|
"kobalt:Semantics:Rhetorical Expressions": {
|
||
|
|
"accuracy": 0.3333333333333333,
|
||
|
|
"correct": 1,
|
||
|
|
"total": 3,
|
||
|
|
"accuracy_percent": 33.3333
|
||
|
|
},
|
||
|
|
"kobalt:Semantics:Semantic Compatibility": {
|
||
|
|
"accuracy": 0.14285714285714285,
|
||
|
|
"correct": 1,
|
||
|
|
"total": 7,
|
||
|
|
"accuracy_percent": 14.2857
|
||
|
|
},
|
||
|
|
"kobalt:Semantics:Word Relationships": {
|
||
|
|
"accuracy": 0.13636363636363635,
|
||
|
|
"correct": 3,
|
||
|
|
"total": 22,
|
||
|
|
"accuracy_percent": 13.6364
|
||
|
|
},
|
||
|
|
"kobalt:Syntax:Agreement": {
|
||
|
|
"accuracy": 0.09803921568627451,
|
||
|
|
"correct": 5,
|
||
|
|
"total": 51,
|
||
|
|
"accuracy_percent": 9.8039
|
||
|
|
},
|
||
|
|
"kobalt:Syntax:Argument Structure & Valency": {
|
||
|
|
"accuracy": 0.12121212121212122,
|
||
|
|
"correct": 4,
|
||
|
|
"total": 33,
|
||
|
|
"accuracy_percent": 12.1212
|
||
|
|
},
|
||
|
|
"kobalt:Syntax:Embedded Clauses": {
|
||
|
|
"accuracy": 0.041666666666666664,
|
||
|
|
"correct": 1,
|
||
|
|
"total": 24,
|
||
|
|
"accuracy_percent": 4.1667
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"kopiqa": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/kopiqa.json",
|
||
|
|
"accuracy": 0.49,
|
||
|
|
"accuracy_percent": 49.0,
|
||
|
|
"correct": 98,
|
||
|
|
"total": 200,
|
||
|
|
"score": "avg_logprob",
|
||
|
|
"categories": {
|
||
|
|
"ko-piqa:1.0": {
|
||
|
|
"accuracy": 0.5681818181818182,
|
||
|
|
"correct": 25,
|
||
|
|
"total": 44,
|
||
|
|
"accuracy_percent": 56.8182
|
||
|
|
},
|
||
|
|
"ko-piqa:None": {
|
||
|
|
"accuracy": 0.46794871794871795,
|
||
|
|
"correct": 73,
|
||
|
|
"total": 156,
|
||
|
|
"accuracy_percent": 46.7949
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"korean_mini_mcq": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/korean_mini_mcq.json",
|
||
|
|
"accuracy": 0.6,
|
||
|
|
"accuracy_percent": 60.0,
|
||
|
|
"correct": 12,
|
||
|
|
"total": 20,
|
||
|
|
"score": "avg_logprob",
|
||
|
|
"categories": {
|
||
|
|
"commonsense": {
|
||
|
|
"accuracy": 0.3333333333333333,
|
||
|
|
"correct": 1,
|
||
|
|
"total": 3,
|
||
|
|
"accuracy_percent": 33.3333
|
||
|
|
},
|
||
|
|
"culture": {
|
||
|
|
"accuracy": 1.0,
|
||
|
|
"correct": 1,
|
||
|
|
"total": 1,
|
||
|
|
"accuracy_percent": 100.0
|
||
|
|
},
|
||
|
|
"instruction": {
|
||
|
|
"accuracy": 1.0,
|
||
|
|
"correct": 3,
|
||
|
|
"total": 3,
|
||
|
|
"accuracy_percent": 100.0
|
||
|
|
},
|
||
|
|
"logic": {
|
||
|
|
"accuracy": 0.6666666666666666,
|
||
|
|
"correct": 2,
|
||
|
|
"total": 3,
|
||
|
|
"accuracy_percent": 66.6667
|
||
|
|
},
|
||
|
|
"math": {
|
||
|
|
"accuracy": 0.3333333333333333,
|
||
|
|
"correct": 1,
|
||
|
|
"total": 3,
|
||
|
|
"accuracy_percent": 33.3333
|
||
|
|
},
|
||
|
|
"reading": {
|
||
|
|
"accuracy": 0.6666666666666666,
|
||
|
|
"correct": 2,
|
||
|
|
"total": 3,
|
||
|
|
"accuracy_percent": 66.6667
|
||
|
|
},
|
||
|
|
"safety": {
|
||
|
|
"accuracy": 0.5,
|
||
|
|
"correct": 1,
|
||
|
|
"total": 2,
|
||
|
|
"accuracy_percent": 50.0
|
||
|
|
},
|
||
|
|
"summary": {
|
||
|
|
"accuracy": 0.5,
|
||
|
|
"correct": 1,
|
||
|
|
"total": 2,
|
||
|
|
"accuracy_percent": 50.0
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"winogrande_ko": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/winogrande_ko.json",
|
||
|
|
"accuracy": 0.455,
|
||
|
|
"accuracy_percent": 45.5,
|
||
|
|
"correct": 91,
|
||
|
|
"total": 200,
|
||
|
|
"score": "avg_logprob",
|
||
|
|
"categories": {
|
||
|
|
"winogrande-ko": {
|
||
|
|
"accuracy": 0.455,
|
||
|
|
"correct": 91,
|
||
|
|
"total": 200,
|
||
|
|
"accuracy_percent": 45.5
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"boundary_and_forensic_diagnostics": {
|
||
|
|
"passk32_argmax_boundary_pairs": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/argmax_boundary_eval_20260609/clean7020_passk32/metrics.json",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"settings": {
|
||
|
|
"pairs": 418,
|
||
|
|
"top_k": 16,
|
||
|
|
"margin_threshold": 0.55,
|
||
|
|
"with_rank": true
|
||
|
|
},
|
||
|
|
"overall": {
|
||
|
|
"count": 418,
|
||
|
|
"target_top1": 0,
|
||
|
|
"target_top1_rate": 0.0,
|
||
|
|
"target_topk": 327,
|
||
|
|
"target_topk_rate": 0.7822966507177034,
|
||
|
|
"target_gt_negative": 0,
|
||
|
|
"target_gt_negative_rate": 0.0,
|
||
|
|
"target_margin_ge_0": 1,
|
||
|
|
"target_margin_ge_0_rate": 0.0023923444976076554,
|
||
|
|
"target_margin_ge_threshold": 0,
|
||
|
|
"target_margin_ge_threshold_rate": 0.0,
|
||
|
|
"margin_mean": -2.341340279465087,
|
||
|
|
"target_logit_mean": 7.655613219338741,
|
||
|
|
"negative_logit_mean": 9.996953498803828,
|
||
|
|
"target_rank_mean": 516.4282296650717
|
||
|
|
},
|
||
|
|
"by_task": [
|
||
|
|
{
|
||
|
|
"task_type": "clean7020_passk32_inventory_accepted_only_instruction",
|
||
|
|
"count": 314,
|
||
|
|
"target_top1": 0,
|
||
|
|
"target_top1_rate": 0.0,
|
||
|
|
"target_topk": 244,
|
||
|
|
"target_topk_rate": 0.7770700636942676,
|
||
|
|
"target_gt_negative": 0,
|
||
|
|
"target_gt_negative_rate": 0.0,
|
||
|
|
"target_margin_ge_0": 1,
|
||
|
|
"target_margin_ge_0_rate": 0.0031847133757961785,
|
||
|
|
"target_margin_ge_threshold": 0,
|
||
|
|
"target_margin_ge_threshold_rate": 0.0,
|
||
|
|
"margin_mean": -2.4404563174885547,
|
||
|
|
"target_logit_mean": 7.31813992056877,
|
||
|
|
"negative_logit_mean": 9.758596238057326,
|
||
|
|
"target_rank_mean": 409.171974522293
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"task_type": "clean7020_passk32_inventory_accepted_only_math",
|
||
|
|
"count": 69,
|
||
|
|
"target_top1": 0,
|
||
|
|
"target_top1_rate": 0.0,
|
||
|
|
"target_topk": 60,
|
||
|
|
"target_topk_rate": 0.8695652173913043,
|
||
|
|
"target_gt_negative": 0,
|
||
|
|
"target_gt_negative_rate": 0.0,
|
||
|
|
"target_margin_ge_0": 0,
|
||
|
|
"target_margin_ge_0_rate": 0.0,
|
||
|
|
"target_margin_ge_threshold": 0,
|
||
|
|
"target_margin_ge_threshold_rate": 0.0,
|
||
|
|
"margin_mean": -1.7986016757246377,
|
||
|
|
"target_logit_mean": 9.565755208333334,
|
||
|
|
"negative_logit_mean": 11.36435688405797,
|
||
|
|
"target_rank_mean": 26.942028985507246
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"task_type": "clean7020_passk32_inventory_accepted_only_reasoning",
|
||
|
|
"count": 35,
|
||
|
|
"target_top1": 0,
|
||
|
|
"target_top1_rate": 0.0,
|
||
|
|
"target_topk": 23,
|
||
|
|
"target_topk_rate": 0.6571428571428571,
|
||
|
|
"target_gt_negative": 0,
|
||
|
|
"target_gt_negative_rate": 0.0,
|
||
|
|
"target_margin_ge_0": 0,
|
||
|
|
"target_margin_ge_0_rate": 0.0,
|
||
|
|
"target_margin_ge_threshold": 0,
|
||
|
|
"target_margin_ge_threshold_rate": 0.0,
|
||
|
|
"margin_mean": -2.5220982142857142,
|
||
|
|
"target_logit_mean": 6.917522321428572,
|
||
|
|
"negative_logit_mean": 9.439620535714285,
|
||
|
|
"target_rank_mean": 2443.657142857143
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"gspo_scoreseq_parent_boundary_pairs": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/clean7020_gspo_scoreseq_v1_boundary_parent7020.json",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"settings": {
|
||
|
|
"pairs": 33226,
|
||
|
|
"top_k": 5,
|
||
|
|
"margin_threshold": 0.65,
|
||
|
|
"with_rank": true
|
||
|
|
},
|
||
|
|
"overall": {
|
||
|
|
"count": 33226,
|
||
|
|
"target_top1": 588,
|
||
|
|
"target_top1_rate": 0.017696984289411907,
|
||
|
|
"target_topk": 7844,
|
||
|
|
"target_topk_rate": 0.23608017817371937,
|
||
|
|
"target_gt_negative": 662,
|
||
|
|
"target_gt_negative_rate": 0.019924155781616805,
|
||
|
|
"target_margin_ge_0": 679,
|
||
|
|
"target_margin_ge_0_rate": 0.020435803286582798,
|
||
|
|
"target_margin_ge_threshold": 70,
|
||
|
|
"target_margin_ge_threshold_rate": 0.002106783843977608,
|
||
|
|
"margin_mean": -4.19784176139393,
|
||
|
|
"target_logit_mean": 4.610714955296313,
|
||
|
|
"negative_logit_mean": 8.808556716690243,
|
||
|
|
"target_rank_mean": 1509.9462168181544
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"trainonly_grounded_bridge_forensics_shard00": {
|
||
|
|
"source_json": "/tmp/chojm_hw2_runs/eval/trainonly_grounded_bridge6990_vs_clean7020_20260609/clean7020_shard00/generation-forensics.json",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"summary": {
|
||
|
|
"greedy|plain": {
|
||
|
|
"rows": 128,
|
||
|
|
"bad": 128,
|
||
|
|
"bad_rate": 1.0,
|
||
|
|
"eos_hit": 36,
|
||
|
|
"eos_hit_rate": 0.28125,
|
||
|
|
"semantic_hit": 0,
|
||
|
|
"semantic_hit_rate": 0.0,
|
||
|
|
"avg_forensic_score": 31.9921875,
|
||
|
|
"flags": {
|
||
|
|
"semantic_weak_miss": 128,
|
||
|
|
"max_new_tokens_hit": 92,
|
||
|
|
"token_3gram_repetition": 99,
|
||
|
|
"char_loop": 37,
|
||
|
|
"punctuation_heavy": 38,
|
||
|
|
"low_hangul_alnum": 35,
|
||
|
|
"role_leak": 1
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"public_comparison_and_replacement_references": {
|
||
|
|
"benchmark_results_file": "benchmark_results_20260609.json",
|
||
|
|
"replacement_evidence_file": "replacement_evidence_20260609.json",
|
||
|
|
"benchmark_results_summary": {
|
||
|
|
"metric": "Korean semantic pass@33",
|
||
|
|
"prompt_set": "/tmp/chojm_hw2_runs/data/clean7020_passk32_accepted_imr_20260609/sentinel_imr_30x3_90.jsonl",
|
||
|
|
"decoding": {
|
||
|
|
"include_greedy_rollout": true,
|
||
|
|
"samples_per_prompt": 32,
|
||
|
|
"temperature": 0.7,
|
||
|
|
"top_p": 0.95,
|
||
|
|
"top_k": 50,
|
||
|
|
"max_new_tokens": 64
|
||
|
|
},
|
||
|
|
"models": {
|
||
|
|
"Jumini-Ko-1.2B-clean7020": {
|
||
|
|
"params": 1260505088,
|
||
|
|
"language_focus": "Korean from-scratch",
|
||
|
|
"summary_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_imr_k32_gpu2/passk_summary.json",
|
||
|
|
"pass_any": "38/90",
|
||
|
|
"pass_any_score": 42.22222222222222,
|
||
|
|
"pass_rollouts": "131/2884",
|
||
|
|
"pass_rollout_score": 4.542302357836339,
|
||
|
|
"greedy_hit": "0/90",
|
||
|
|
"categories": {
|
||
|
|
"instruction": "26/30",
|
||
|
|
"math": "8/30",
|
||
|
|
"reasoning": "4/30"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"TinyLlama-1.1B-intermediate-step-1431k-3T": {
|
||
|
|
"repo_id": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
|
||
|
|
"params": 1100000000,
|
||
|
|
"language_focus": "English general base",
|
||
|
|
"summary_json": "/tmp/chojm_hw2_runs/hf_baseline_compare_20260609/tinyllama_1p1b_base_k32/summary.json",
|
||
|
|
"pass_any": "31/90",
|
||
|
|
"pass_any_score": 34.44444444444444,
|
||
|
|
"pass_rollouts": "56/2966",
|
||
|
|
"pass_rollout_score": 1.8880647336480108,
|
||
|
|
"greedy_hit": "0/90",
|
||
|
|
"categories": {
|
||
|
|
"instruction": "27/30",
|
||
|
|
"math": "4/30",
|
||
|
|
"reasoning": "0/30"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"replacement_summary": {
|
||
|
|
"name": "clean7020",
|
||
|
|
"checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt",
|
||
|
|
"export_dir": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32",
|
||
|
|
"strat240_pass_any": "21/240",
|
||
|
|
"strat240_pass_rollouts": "29/2105",
|
||
|
|
"strat240_mechanical_bad_rollouts": "856/2105",
|
||
|
|
"k32_pass_any": "38/90",
|
||
|
|
"strict_heldout_stable_semantic_hits": "2/12"
|
||
|
|
},
|
||
|
|
"previous_hf_reference": {
|
||
|
|
"name": "v3a6200",
|
||
|
|
"remote_sha_before_replace": "369600b6f14950e4bf6c7c43457ed3ee0921663c",
|
||
|
|
"checkpoint": "/raid2/chojm/hw2_llm_from_scratch/experiments/jumini-ko-1.2b-phase2-mixture-v3a-retention-continue-5700-to-6200-2gpu12-mb9/step_00006200/model.pt",
|
||
|
|
"strat240_pass_any": "0/240",
|
||
|
|
"strat240_pass_rollouts": "0/2094",
|
||
|
|
"strat240_mechanical_bad_rollouts": "1755/2094"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"omission_policy": [
|
||
|
|
"No row-level prompt text or completions are included in this public score appendix.",
|
||
|
|
"Non-exported descendants, row-edit probes, merge/soup probes, and alternate 7020-lineage diagnostics are not presented as scores for the current HF checkpoint.",
|
||
|
|
"External baseline comparison scores remain in benchmark_results_20260609.json and are not duplicated here except by reference."
|
||
|
|
]
|
||
|
|
}
|