From 0d536ff2709a074582d256b6c3c137fba0389b7d Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 7 Jul 2026 20:22:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: properly59/Jumini-Ko-1.2B Source: Original Platform --- .gitattributes | 37 + Jumini-Ko-1.2B-Technical-Report.pdf | 3 + LICENSE | 17 + README.md | 141 +++ benchmark_results_20260609.json | 67 ++ config.json | 32 + generation_config.json | 9 + hf_equivalence_atol2e-5.json | 30 + jumini_config.json | 23 + jumini_internal_scores_20260609.json | 1671 ++++++++++++++++++++++++++ load_smoke.json | 19 + model.safetensors | 3 + replacement_evidence_20260609.json | 32 + special_tokens_map.json | 6 + tokenizer.json | 3 + tokenizer_config.json | 12 + 16 files changed, 2105 insertions(+) create mode 100644 .gitattributes create mode 100644 Jumini-Ko-1.2B-Technical-Report.pdf create mode 100644 LICENSE create mode 100644 README.md create mode 100644 benchmark_results_20260609.json create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 hf_equivalence_atol2e-5.json create mode 100644 jumini_config.json create mode 100644 jumini_internal_scores_20260609.json create mode 100644 load_smoke.json create mode 100644 model.safetensors create mode 100644 replacement_evidence_20260609.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a87a57d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +Jumini-Ko-1.2B-Technical-Report.pdf filter=lfs diff=lfs merge=lfs -text diff --git a/Jumini-Ko-1.2B-Technical-Report.pdf b/Jumini-Ko-1.2B-Technical-Report.pdf new file mode 100644 index 0000000..4de6175 --- /dev/null +++ b/Jumini-Ko-1.2B-Technical-Report.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fd06737ba1b4f7844cd9b18cb9eec4b94c5d9c7871c80407d9d5687815f9fe8 +size 691101 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..852d45c --- /dev/null +++ b/LICENSE @@ -0,0 +1,17 @@ +Apache License +Version 2.0, January 2004 +https://www.apache.org/licenses/ + +Copyright 2026 Jumini-Ko HW2 contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..5b4d957 --- /dev/null +++ b/README.md @@ -0,0 +1,141 @@ +--- +language: +- ko +license: apache-2.0 +library_name: transformers +pipeline_tag: text-generation +tags: +- korean +- causal-lm +- decoder-only +- from-scratch +- instruction-tuned +- 1.2b +model-index: +- name: Jumini-Ko-1.2B + results: + - task: {type: text-generation, name: Korean Knowledge (HAE-RAE Bench)} + dataset: {type: HAERAE-HUB/HAE_RAE_BENCH_1.0, name: HAE-RAE Bench} + metrics: + - {type: acc, name: accuracy (5-shot), value: 21.9} + - task: {type: text-generation, name: Korean Reading (Belebele-Ko)} + dataset: {type: facebook/belebele, name: Belebele (kor_Hang)} + metrics: + - {type: acc, name: accuracy (5-shot), value: 27.9} + - task: {type: text-generation, name: KMMLU} + dataset: {type: HAERAE-HUB/KMMLU, name: KMMLU} + metrics: + - {type: acc, name: accuracy (5-shot), value: 24.3} + - task: {type: text-generation, name: KoBEST} + dataset: {type: skt/kobest_v1, name: KoBEST} + metrics: + - {type: acc, name: accuracy (5-shot), value: 49.5} +--- + +# Jumini-Ko-1.2B + +**Jumini-Ko-1.2B** is a 1.26B-parameter Korean decoder-only language model **trained from +scratch** β€” its architecture, tokenizer, data pipeline, and training loop were all built +in-house, and it is *not* a fine-tune of any existing model. It is a compact, +Korean-specialized model designed to run on commodity hardware. + +> Among the evaluated **open non-flagship Korean baselines** (`polyglot-ko-1.3b`, `Tri-1.9B`), +> Jumini-Ko-1.2B is the **strongest on Korean knowledge (HAE-RAE) and reading comprehension +> (Belebele-Ko)** β€” despite being the **smallest** model compared. The flagship +> `EXAONE-4.0-1.2B`, trained on far more data/compute, is stronger on all four benchmarks. + +## Highlights + +- πŸ‡°πŸ‡· **Korean-specialized, from scratch** β€” Llama-3-style architecture (RoPE, GQA, SwiGLU, + RMSNorm), 128K byte-level BPE tokenizer, trained from random initialization. +- πŸ₯‡ **Beats the size-matched `polyglot-ko-1.3b` and the larger `Tri-1.9B`** on HAE-RAE and + Belebele-Ko (5-shot), the two Korean-language benchmarks emphasized here. (It trails + `polyglot-ko-1.3b` on KoBEST commonsense and KMMLU, and the flagship `EXAONE-4.0-1.2B` overall.) +- πŸ”¬ **A data-centric recipe** β€” we show that *which* corpus you continue-pretrain on decides + *which* capability improves (web β†’ commonsense, Wikipedia β†’ knowledge). +- πŸ“¦ **Edge-friendly** β€” 1.26B parameters; runs comfortably on a single consumer GPU. + +## Benchmark Results + +Korean benchmarks via the EleutherAI `lm-evaluation-harness`, 5-shot, accuracy (%). All models +evaluated under identical settings. **Bold** = best, underline = second best. + +| Benchmark | **Jumini-Ko-1.2B** (1.26B) | polyglot-ko-1.3b (1.43B) | Tri-1.9B (1.9B) | EXAONE-4.0-1.2B† (1.28B) | +|---|:--:|:--:|:--:|:--:| +| HAE-RAE (Korean knowledge) | 21.9 | 18.7 | 18.9 | **30.0** | +| Belebele-Ko (reading) | 27.9 | 22.4 | 22.9 | **44.7** | +| KMMLU (knowledge) | 24.3 | 27.8 | 16.6 | **32.6** | +| KoBEST (commonsense) | 49.5 | **55.9** | 50.1 | 50.6 | + +† EXAONE-4.0-1.2B is a strong flagship model trained on vastly more data/compute, shown as +an aspirational reference. Against the **open same-tier** baselines (polyglot-ko-1.3b, Tri-1.9B), +Jumini leads on the Korean-specific HAE-RAE and Belebele-Ko while being the smallest model. + +Jumini also beats `polyglot-ko-1.3b` on **4 of 5 HAE-RAE subtasks** (history, loan-word, +rare-word, standard-nomenclature). It trails `polyglot-ko-1.3b` on commonsense (KoBEST) and broad +knowledge (KMMLU). Full per-subtask numbers are in the technical report. + +## Quickstart + +```python +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +repo = "properly59/Jumini-Ko-1.2B" +tok = AutoTokenizer.from_pretrained(repo) +model = AutoModelForCausalLM.from_pretrained(repo, torch_dtype=torch.float16, device_map="auto") + +prompt = "### 질문:\nλŒ€ν•œλ―Όκ΅­μ˜ μˆ˜λ„λŠ” μ–΄λ””μΈκ°€μš”?\n\n### λ‹΅λ³€:\n" +ids = tok(tok.bos_token + prompt, return_tensors="pt", add_special_tokens=False).to(model.device) +out = model.generate(**ids, max_new_tokens=128, do_sample=True, temperature=0.8, + min_p=0.05, repetition_penalty=1.2, no_repeat_ngram_size=3, + pad_token_id=tok.pad_token_id) +print(tok.decode(out[0][ids.input_ids.shape[1]:], skip_special_tokens=True)) +``` + +## Model Details + +| | | +|---|---| +| Architecture | Decoder-only Transformer (Llama-3 family) | +| Parameters | 1.26B (hidden 2048, 28 layers, 32 Q / 8 KV heads, SwiGLU 4096) | +| Position encoding | RoPE (ΞΈ = 500,000) | +| Tokenizer | Byte-level BPE, 128,000 vocab | +| Context length | 4,096 | +| Precision | bf16 / fp16 | +| License | Apache-2.0 | + +## Training + +A three-stage, fully-documented pipeline on top of the from-scratch base: + +1. **Continued pre-training** on a high-quality Korean mixture (FineWeb-2 `kor_Hang`, + KOREAN-WEBTEXT, Korean Wikipedia), document-boundary packed. +2. **Encyclopedic annealing** on Korean Wikipedia (LR β†’ 0) β€” the most token-efficient route to + Korean knowledge. +3. **Supervised fine-tuning** on a 132K permissively-licensed Korean instruction mixture + (KoAlpaca, OpenOrca-KO, KOpen-Platypus, KULLM-v2), with completion-only loss and explicit EOS + supervision. + +All continued-pretraining and instruction data are public corpora used only for post-training; +no external pretrained weights are used. A benchmark decontamination check found **0.00%** of benchmark +items substantially covered (β‰₯50% of 25-character shingles) by the instruction data. + +## Intended Use & Limitations + +Intended for Korean text generation, QA, summarization, and research on small-model training. +As a compact model trained from scratch under a constrained budget, its **factual accuracy is +limited** and it can produce incorrect content; greedy decoding is best paired with a repetition +penalty. It trails much larger / higher-budget Korean models (e.g., EXAONE) on knowledge tasks +and has not undergone safety alignment. Use for research and non-critical applications only. + +## Citation + +```bibtex +@techreport{jumini2026, + title = {Jumini-Ko-1.2B Technical Report}, + author = {Cho, Ju-min}, + year = {2026}, + note = {https://huggingface.co/properly59/Jumini-Ko-1.2B} +} +``` diff --git a/benchmark_results_20260609.json b/benchmark_results_20260609.json new file mode 100644 index 0000000..06857ba --- /dev/null +++ b/benchmark_results_20260609.json @@ -0,0 +1,67 @@ +{ + "benchmark_date": "2026-06-09", + "summary": "Local train-only Korean semantic sentinel diagnostics. These are not official leaderboard scores.", + "jumini_internal_scores_file": "jumini_internal_scores_20260609.json", + "main_public_comparison": { + "metric": "Korean semantic pass@33", + "prompt_set": "/tmp/chojm_hw2_runs/data/clean7020_passk32_accepted_imr_20260609/sentinel_imr_30x3_90.jsonl", + "decoding": { + "include_greedy_rollout": true, + "samples_per_prompt": 32, + "temperature": 0.7, + "top_p": 0.95, + "top_k": 50, + "max_new_tokens": 64 + }, + "models": { + "Jumini-Ko-1.2B-clean7020": { + "params": 1260505088, + "language_focus": "Korean from-scratch", + "summary_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_imr_k32_gpu2/passk_summary.json", + "pass_any": "38/90", + "pass_any_score": 42.22222222222222, + "pass_rollouts": "131/2884", + "pass_rollout_score": 4.542302357836339, + "greedy_hit": "0/90", + "categories": { + "instruction": "26/30", + "math": "8/30", + "reasoning": "4/30" + } + }, + "TinyLlama-1.1B-intermediate-step-1431k-3T": { + "repo_id": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T", + "params": 1100000000, + "language_focus": "English general base", + "summary_json": "/tmp/chojm_hw2_runs/hf_baseline_compare_20260609/tinyllama_1p1b_base_k32/summary.json", + "pass_any": "31/90", + "pass_any_score": 34.44444444444444, + "pass_rollouts": "56/2966", + "pass_rollout_score": 1.8880647336480108, + "greedy_hit": "0/90", + "categories": { + "instruction": "27/30", + "math": "4/30", + "reasoning": "0/30" + } + } + } + }, + "public_refresh_evidence": { + "metric": "Stratified Korean semantic pass@k", + "previous_hf_v3a6200": { + "summary_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/v3a6200_strat240/passk_summary.json", + "pass_any": "0/240", + "pass_any_score": 0.0, + "pass_rollouts": "0/2094", + "pass_rollout_score": 0.0 + }, + "current_clean7020": { + "summary_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_strat240_gpu2/passk_summary.json", + "pass_any": "21/240", + "pass_any_score": 8.75, + "pass_rollouts": "29/2105", + "pass_rollout_score": 1.3776722090261282 + } + } +} diff --git a/config.json b/config.json new file mode 100644 index 0000000..093ad60 --- /dev/null +++ b/config.json @@ -0,0 +1,32 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "dtype": "bfloat16", + "eos_token_id": 2, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 4096, + "max_position_embeddings": 4096, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 0, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_parameters": { + "rope_theta": 500000.0, + "rope_type": "default" + }, + "tie_word_embeddings": true, + "transformers_version": "5.12.0", + "use_cache": false, + "vocab_size": 128000 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..b6a126f --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "bos_token_id": 1, + "do_sample": true, + "eos_token_id": 2, + "pad_token_id": 0, + "temperature": 0.8, + "top_k": 50, + "transformers_version": "5.12.0" +} diff --git a/hf_equivalence_atol2e-5.json b/hf_equivalence_atol2e-5.json new file mode 100644 index 0000000..a4aba60 --- /dev/null +++ b/hf_equivalence_atol2e-5.json @@ -0,0 +1,30 @@ +{ + "ok": true, + "logits_shape": [ + 2, + 8, + 128000 + ], + "max_abs_diff": 1.4781951904296875e-05, + "mean_abs_diff": 1.073120984074194e-06, + "rms_diff": 1.5380918512164499e-06, + "max_rel_diff": 0.04641776159405708, + "argmax_mismatch_rate": 0.0, + "atol": 2e-05, + "rtol": 2e-05, + "config": "/raid2/chojm/hw2_llm_from_scratch/configs/jumini_1_2b.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "hf_model_dir": "/raid2/chojm/hw2_llm_from_scratch/models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32", + "hf_model_class": "LlamaForCausalLM", + "device": "cpu", + "comparison_dtype": "float32", + "hf_export_torch_dtype": "float32", + "round_native_to_export_dtype": false, + "input_shape": [ + 2, + 8 + ], + "input_ids_sha256": "b9225a84aaeeb49dbb7ac34866e25a8fdbf0823d9fe02530e3686d2e09f72f6f", + "seed": 1234, + "max_tokens": 64 +} diff --git a/jumini_config.json b/jumini_config.json new file mode 100644 index 0000000..4c7bb36 --- /dev/null +++ b/jumini_config.json @@ -0,0 +1,23 @@ +{ + "vocab_size": 128000, + "hidden_size": 2048, + "intermediate_size": 4096, + "num_hidden_layers": 28, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "max_position_embeddings": 4096, + "rope_theta": 500000.0, + "rms_norm_eps": 1e-05, + "initializer_range": 0.02, + "tie_word_embeddings": true, + "attention_dropout": 0.0, + "residual_dropout": 0.0, + "gradient_checkpointing": true, + "pad_token_id": 0, + "bos_token_id": 1, + "eos_token_id": 2, + "model_type": "jumini", + "architectures": [ + "JuminiForCausalLM" + ] +} diff --git a/jumini_internal_scores_20260609.json b/jumini_internal_scores_20260609.json new file mode 100644 index 0000000..152edbe --- /dev/null +++ b/jumini_internal_scores_20260609.json @@ -0,0 +1,1671 @@ +{ + "created_at": "2026-06-09T10:22:09.271056+00:00", + "repo_id": "properly59/Jumini-Ko-1.2B", + "model_card_scope": "Completed score summaries for the current public clean7020 checkpoint. Row-level prompts/completions and non-exported descendant checkpoints are intentionally omitted.", + "current_public_checkpoint": { + "name": "clean7020", + "native_checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "export_dir": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32", + "source_training_step": 7020, + "tokens_seen": 1025474560, + "hf_remote_sha_before_this_appendix": "82815c2b339f8674792b8a5910c3eb913aab9e60" + }, + "validation": { + "transformers_load_smoke": { + "source_json": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32/load_smoke.json", + "ok": true, + "model_class": "LlamaForCausalLM", + "parameter_count": 1260505088, + "tokenizer_len": 128000 + }, + "hf_export_equivalence": { + "source_json": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32/hf_equivalence_atol2e-5.json", + "ok": true, + "atol": 2e-05, + "rtol": 2e-05, + "max_abs_diff": 1.4781951904296875e-05, + "mean_abs_diff": 1.073120984074194e-06, + "rms_diff": 1.5380918512164499e-06, + "argmax_mismatch_rate": 0.0 + }, + "native_checkpoint_summary": { + "source_json": "/tmp/chojm_hw2_runs/eval/argmax_boundary_layernormshock_rerun20260608_gate/step_00007020/checkpoint-summary.json", + "ok": true, + "step": 7020, + "tokens_seen": 1025474560, + "file_size_bytes": 6090713562, + "native_tensor_parameter_count": 1522649088 + } + }, + "strict_heldout_project_gate": { + "source_json": "/tmp/chojm_hw2_runs/eval/argmax_boundary_layernormshock_rerun20260608_gate/step_00007020/heldout-semantic-gate.json", + "preset": "greedy|plain", + "rows": 12, + "semantic_hits": { + "count": 2, + "total": 12, + "rate": 0.16666666666666666, + "percent": 16.6667 + }, + "stable_semantic_hits": { + "count": 2, + "total": 12, + "rate": 0.16666666666666666, + "percent": 16.6667 + }, + "eos_hits": { + "count": 5, + "total": 12, + "rate": 0.4166666666666667, + "percent": 41.6667 + }, + "max_new_tokens_hits": { + "count": 7, + "total": 12, + "rate": 0.5833333333333334, + "percent": 58.3333 + }, + "mechanical_bad": { + "count": 9, + "total": 12, + "rate": 0.75, + "percent": 75.0 + }, + "empty": { + "count": 1, + "total": 12, + "rate": 0.08333333333333333, + "percent": 8.3333 + }, + "semantic_hit_ids": [ + "heldout_summary_battery_001", + "heldout_minutes_split_001" + ], + "flag_counts": { + "max_new_tokens_hit": 7, + "token_3gram_repetition": 8, + "punctuation_heavy": 3, + "char_loop": 3, + "low_hangul_alnum": 3, + "empty": 1 + } + }, + "korean_semantic_passk_and_decode_diagnostics": { + "k32_korean_semantic_sentinel_90": { + "label": "90-prompt Korean semantic sentinel, greedy + 32 sampled rollouts", + "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_imr_k32_gpu2/passk_summary.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "settings": { + "eos_logit_bias": 0.0, + "include_greedy_rollout": true, + "max_new_tokens": 64, + "no_repeat_ngram_size": 0, + "precision": "fp16", + "prompt_variant": "plain", + "repetition_penalty": 1.0, + "samples_per_prompt": 32, + "seed": 20260609, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.95 + }, + "totals": { + "evaluated_prompts": 90, + "read_prompts": 90, + "rollouts": 2884, + "pass_any_prompts": 38, + "pass_any_rate": 0.4222222222222222, + "pass_rollouts": 131, + "pass_rollout_rate": 0.045423023578363385, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 38, + "sample_hit_rate": 0.4222222222222222, + "eos_hits": 2071, + "eos_hit_rate": 0.7180998613037448, + "mechanical_bad_rollouts": 930, + "mechanical_bad_rate": 0.3224687933425798, + "pass_any_categories": [ + "instruction", + "math", + "reasoning" + ], + "mechanical_flags": { + "char_loop": 222, + "punctuation_heavy": 226, + "role_leak": 8, + "token_3gram_repetition": 914 + } + }, + "by_category": { + "instruction": { + "rollouts": 913, + "pass_any_prompts": 26, + "pass_any_rate": 0.8666666666666667, + "pass_rollouts": 110, + "pass_rollout_rate": 0.12048192771084337, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 26, + "sample_hit_rate": 0.8666666666666667, + "eos_hits": 561, + "eos_hit_rate": 0.6144578313253012, + "mechanical_bad_rollouts": 384, + "mechanical_bad_rate": 0.42059145673603504, + "mechanical_flags": { + "char_loop": 152, + "punctuation_heavy": 161, + "token_3gram_repetition": 373 + }, + "prompts": 30 + }, + "math": { + "rollouts": 986, + "pass_any_prompts": 8, + "pass_any_rate": 0.26666666666666666, + "pass_rollouts": 14, + "pass_rollout_rate": 0.014198782961460446, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 8, + "sample_hit_rate": 0.26666666666666666, + "eos_hits": 743, + "eos_hit_rate": 0.7535496957403651, + "mechanical_bad_rollouts": 311, + "mechanical_bad_rate": 0.3154158215010142, + "mechanical_flags": { + "char_loop": 18, + "punctuation_heavy": 15, + "role_leak": 8, + "token_3gram_repetition": 306 + }, + "prompts": 30 + }, + "reasoning": { + "rollouts": 985, + "pass_any_prompts": 4, + "pass_any_rate": 0.13333333333333333, + "pass_rollouts": 7, + "pass_rollout_rate": 0.007106598984771574, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 4, + "sample_hit_rate": 0.13333333333333333, + "eos_hits": 767, + "eos_hit_rate": 0.7786802030456853, + "mechanical_bad_rollouts": 235, + "mechanical_bad_rate": 0.23857868020304568, + "mechanical_flags": { + "char_loop": 52, + "punctuation_heavy": 50, + "token_3gram_repetition": 235 + }, + "prompts": 30 + } + } + }, + "stratified_korean_semantic_sentinel_240": { + "label": "240-prompt stratified Korean semantic sentinel", + "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_strat240_gpu2/passk_summary.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "settings": { + "eos_logit_bias": 0.0, + "include_greedy_rollout": true, + "max_new_tokens": 64, + "no_repeat_ngram_size": 0, + "precision": "fp16", + "prompt_variant": "plain", + "repetition_penalty": 1.0, + "samples_per_prompt": 8, + "seed": 20260609, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.95 + }, + "totals": { + "evaluated_prompts": 240, + "read_prompts": 240, + "rollouts": 2105, + "pass_any_prompts": 21, + "pass_any_rate": 0.0875, + "pass_rollouts": 29, + "pass_rollout_rate": 0.013776722090261283, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 21, + "sample_hit_rate": 0.0875, + "eos_hits": 1238, + "eos_hit_rate": 0.58812351543943, + "mechanical_bad_rollouts": 856, + "mechanical_bad_rate": 0.4066508313539192, + "pass_any_categories": [ + "instruction", + "math", + "reasoning" + ], + "mechanical_flags": { + "char_loop": 230, + "punctuation_heavy": 214, + "role_leak": 17, + "token_3gram_repetition": 838 + } + }, + "by_category": { + "causal": { + "rollouts": 262, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 122, + "eos_hit_rate": 0.46564885496183206, + "mechanical_bad_rollouts": 156, + "mechanical_bad_rate": 0.5954198473282443, + "mechanical_flags": { + "char_loop": 37, + "punctuation_heavy": 35, + "token_3gram_repetition": 156 + }, + "prompts": 30 + }, + "compare": { + "rollouts": 265, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 155, + "eos_hit_rate": 0.5849056603773585, + "mechanical_bad_rollouts": 124, + "mechanical_bad_rate": 0.4679245283018868, + "mechanical_flags": { + "char_loop": 25, + "punctuation_heavy": 25, + "role_leak": 3, + "token_3gram_repetition": 122 + }, + "prompts": 30 + }, + "instruction": { + "rollouts": 255, + "pass_any_prompts": 17, + "pass_any_rate": 0.5666666666666667, + "pass_rollouts": 24, + "pass_rollout_rate": 0.09411764705882353, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 17, + "sample_hit_rate": 0.5666666666666667, + "eos_hits": 149, + "eos_hit_rate": 0.5843137254901961, + "mechanical_bad_rollouts": 119, + "mechanical_bad_rate": 0.4666666666666667, + "mechanical_flags": { + "char_loop": 64, + "punctuation_heavy": 66, + "token_3gram_repetition": 116 + }, + "prompts": 30 + }, + "math": { + "rollouts": 270, + "pass_any_prompts": 2, + "pass_any_rate": 0.06666666666666667, + "pass_rollouts": 2, + "pass_rollout_rate": 0.007407407407407408, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 2, + "sample_hit_rate": 0.06666666666666667, + "eos_hits": 207, + "eos_hit_rate": 0.7666666666666667, + "mechanical_bad_rollouts": 92, + "mechanical_bad_rate": 0.34074074074074073, + "mechanical_flags": { + "char_loop": 4, + "punctuation_heavy": 3, + "role_leak": 4, + "token_3gram_repetition": 90 + }, + "prompts": 30 + }, + "notice": { + "rollouts": 270, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 71, + "eos_hit_rate": 0.26296296296296295, + "mechanical_bad_rollouts": 99, + "mechanical_bad_rate": 0.36666666666666664, + "mechanical_flags": { + "char_loop": 19, + "punctuation_heavy": 8, + "token_3gram_repetition": 96 + }, + "prompts": 30 + }, + "reasoning": { + "rollouts": 269, + "pass_any_prompts": 2, + "pass_any_rate": 0.06666666666666667, + "pass_rollouts": 3, + "pass_rollout_rate": 0.011152416356877323, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 2, + "sample_hit_rate": 0.06666666666666667, + "eos_hits": 207, + "eos_hit_rate": 0.7695167286245354, + "mechanical_bad_rollouts": 59, + "mechanical_bad_rate": 0.21933085501858737, + "mechanical_flags": { + "char_loop": 7, + "punctuation_heavy": 9, + "token_3gram_repetition": 59 + }, + "prompts": 30 + }, + "safety": { + "rollouts": 258, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 156, + "eos_hit_rate": 0.6046511627906976, + "mechanical_bad_rollouts": 111, + "mechanical_bad_rate": 0.43023255813953487, + "mechanical_flags": { + "char_loop": 30, + "punctuation_heavy": 29, + "role_leak": 9, + "token_3gram_repetition": 104 + }, + "prompts": 30 + }, + "style": { + "rollouts": 256, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 171, + "eos_hit_rate": 0.66796875, + "mechanical_bad_rollouts": 96, + "mechanical_bad_rate": 0.375, + "mechanical_flags": { + "char_loop": 44, + "punctuation_heavy": 39, + "role_leak": 1, + "token_3gram_repetition": 95 + }, + "prompts": 30 + } + } + }, + "k3_plain64_baseline_90": { + "label": "90-prompt Korean semantic sentinel, greedy + 3 sampled rollouts", + "source_json": "/tmp/chojm_hw2_runs/checkpoint_interpolation_20260609/clean7020_baseline/passk_imr_k3_plain64/passk_summary.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "settings": { + "eos_logit_bias": 0.0, + "include_greedy_rollout": true, + "max_new_tokens": 64, + "no_repeat_ngram_size": 0, + "precision": "fp16", + "prompt_variant": "plain", + "repetition_penalty": 1.0, + "samples_per_prompt": 3, + "seed": 20260609, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.95 + }, + "totals": { + "evaluated_prompts": 90, + "read_prompts": 90, + "rollouts": 357, + "pass_any_prompts": 11, + "pass_any_rate": 0.12222222222222222, + "pass_rollouts": 11, + "pass_rollout_rate": 0.03081232492997199, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 11, + "sample_hit_rate": 0.12222222222222222, + "eos_hits": 224, + "eos_hit_rate": 0.6274509803921569, + "mechanical_bad_rollouts": 163, + "mechanical_bad_rate": 0.4565826330532213, + "pass_any_categories": [ + "instruction", + "math" + ], + "mechanical_flags": { + "char_loop": 56, + "punctuation_heavy": 56, + "token_3gram_repetition": 161 + } + }, + "by_category": { + "instruction": { + "rollouts": 119, + "pass_any_prompts": 10, + "pass_any_rate": 0.3333333333333333, + "pass_rollouts": 10, + "pass_rollout_rate": 0.08403361344537816, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 10, + "sample_hit_rate": 0.3333333333333333, + "eos_hits": 58, + "eos_hit_rate": 0.48739495798319327, + "mechanical_bad_rollouts": 67, + "mechanical_bad_rate": 0.5630252100840336, + "mechanical_flags": { + "char_loop": 44, + "punctuation_heavy": 45, + "token_3gram_repetition": 65 + }, + "prompts": 30 + }, + "math": { + "rollouts": 119, + "pass_any_prompts": 1, + "pass_any_rate": 0.03333333333333333, + "pass_rollouts": 1, + "pass_rollout_rate": 0.008403361344537815, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 1, + "sample_hit_rate": 0.03333333333333333, + "eos_hits": 73, + "eos_hit_rate": 0.6134453781512605, + "mechanical_bad_rollouts": 61, + "mechanical_bad_rate": 0.5126050420168067, + "mechanical_flags": { + "char_loop": 5, + "punctuation_heavy": 3, + "token_3gram_repetition": 61 + }, + "prompts": 30 + }, + "reasoning": { + "rollouts": 119, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 93, + "eos_hit_rate": 0.7815126050420168, + "mechanical_bad_rollouts": 35, + "mechanical_bad_rate": 0.29411764705882354, + "mechanical_flags": { + "char_loop": 7, + "punctuation_heavy": 8, + "token_3gram_repetition": 35 + }, + "prompts": 30 + } + } + }, + "greedy_only_90": { + "label": "90-prompt Korean semantic sentinel, greedy only", + "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_sentinel_imr_greedy_gpu1/passk_summary.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "settings": { + "eos_logit_bias": 0.0, + "include_greedy_rollout": true, + "max_new_tokens": 64, + "no_repeat_ngram_size": 0, + "precision": "fp16", + "prompt_variant": "plain", + "repetition_penalty": 1.0, + "samples_per_prompt": 0, + "seed": 20260610, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.95 + }, + "totals": { + "evaluated_prompts": 90, + "read_prompts": 90, + "rollouts": 90, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 46, + "eos_hit_rate": 0.5111111111111111, + "mechanical_bad_rollouts": 65, + "mechanical_bad_rate": 0.7222222222222222, + "pass_any_categories": [], + "mechanical_flags": { + "char_loop": 24, + "punctuation_heavy": 24, + "token_3gram_repetition": 65 + } + }, + "by_category": { + "instruction": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 8, + "eos_hit_rate": 0.26666666666666666, + "mechanical_bad_rollouts": 22, + "mechanical_bad_rate": 0.7333333333333333, + "mechanical_flags": { + "char_loop": 22, + "punctuation_heavy": 22, + "token_3gram_repetition": 22 + }, + "prompts": 30 + }, + "math": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 15, + "eos_hit_rate": 0.5, + "mechanical_bad_rollouts": 27, + "mechanical_bad_rate": 0.9, + "mechanical_flags": { + "char_loop": 1, + "punctuation_heavy": 1, + "token_3gram_repetition": 27 + }, + "prompts": 30 + }, + "reasoning": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 23, + "eos_hit_rate": 0.7666666666666667, + "mechanical_bad_rollouts": 16, + "mechanical_bad_rate": 0.5333333333333333, + "mechanical_flags": { + "char_loop": 1, + "punctuation_heavy": 1, + "token_3gram_repetition": 16 + }, + "prompts": 30 + } + } + }, + "two_prompt_smoke": { + "label": "2-prompt generation smoke only", + "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/smoke_clean7020/summary.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "settings": { + "eos_logit_bias": 0.0, + "include_greedy_rollout": true, + "max_new_tokens": 16, + "no_repeat_ngram_size": 0, + "precision": "fp16", + "prompt_variant": "plain", + "repetition_penalty": 1.0, + "samples_per_prompt": 1, + "seed": 20260609, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.95 + }, + "totals": { + "evaluated_prompts": 2, + "read_prompts": 2, + "rollouts": 4, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 0, + "eos_hit_rate": 0.0, + "mechanical_bad_rollouts": 1, + "mechanical_bad_rate": 0.25, + "pass_any_categories": [], + "mechanical_flags": { + "token_3gram_repetition": 1 + } + }, + "by_category": { + "compare": { + "rollouts": 2, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 0, + "eos_hit_rate": 0.0, + "mechanical_bad_rollouts": 1, + "mechanical_bad_rate": 0.5, + "mechanical_flags": { + "token_3gram_repetition": 1 + }, + "prompts": 1 + }, + "notice": { + "rollouts": 2, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 0, + "eos_hit_rate": 0.0, + "mechanical_bad_rollouts": 0, + "mechanical_bad_rate": 0.0, + "mechanical_flags": {}, + "prompts": 1 + } + } + }, + "decode_control_eosbias1": { + "label": "90-prompt greedy decode control: eosbias1", + "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_decode_sentinel/eosbias1/passk_summary.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "settings": { + "eos_logit_bias": 1.0, + "include_greedy_rollout": true, + "max_new_tokens": 64, + "no_repeat_ngram_size": 0, + "precision": "fp16", + "prompt_variant": "plain", + "repetition_penalty": 1.0, + "samples_per_prompt": 0, + "seed": 20260610, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.95 + }, + "totals": { + "evaluated_prompts": 90, + "read_prompts": 90, + "rollouts": 90, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 46, + "eos_hit_rate": 0.5111111111111111, + "mechanical_bad_rollouts": 65, + "mechanical_bad_rate": 0.7222222222222222, + "pass_any_categories": [], + "mechanical_flags": { + "char_loop": 24, + "punctuation_heavy": 24, + "token_3gram_repetition": 65 + } + }, + "by_category": { + "instruction": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 8, + "eos_hit_rate": 0.26666666666666666, + "mechanical_bad_rollouts": 22, + "mechanical_bad_rate": 0.7333333333333333, + "mechanical_flags": { + "char_loop": 22, + "punctuation_heavy": 22, + "token_3gram_repetition": 22 + }, + "prompts": 30 + }, + "math": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 15, + "eos_hit_rate": 0.5, + "mechanical_bad_rollouts": 27, + "mechanical_bad_rate": 0.9, + "mechanical_flags": { + "char_loop": 1, + "punctuation_heavy": 1, + "token_3gram_repetition": 27 + }, + "prompts": 30 + }, + "reasoning": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 23, + "eos_hit_rate": 0.7666666666666667, + "mechanical_bad_rollouts": 16, + "mechanical_bad_rate": 0.5333333333333333, + "mechanical_flags": { + "char_loop": 1, + "punctuation_heavy": 1, + "token_3gram_repetition": 16 + }, + "prompts": 30 + } + } + }, + "decode_control_no_repeat3": { + "label": "90-prompt greedy decode control: no_repeat3", + "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_decode_sentinel/no_repeat3/passk_summary.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "settings": { + "eos_logit_bias": 0.0, + "include_greedy_rollout": true, + "max_new_tokens": 64, + "no_repeat_ngram_size": 3, + "precision": "fp16", + "prompt_variant": "plain", + "repetition_penalty": 1.0, + "samples_per_prompt": 0, + "seed": 20260610, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.95 + }, + "totals": { + "evaluated_prompts": 90, + "read_prompts": 90, + "rollouts": 90, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 46, + "eos_hit_rate": 0.5111111111111111, + "mechanical_bad_rollouts": 65, + "mechanical_bad_rate": 0.7222222222222222, + "pass_any_categories": [], + "mechanical_flags": { + "char_loop": 24, + "punctuation_heavy": 24, + "token_3gram_repetition": 65 + } + }, + "by_category": { + "instruction": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 8, + "eos_hit_rate": 0.26666666666666666, + "mechanical_bad_rollouts": 22, + "mechanical_bad_rate": 0.7333333333333333, + "mechanical_flags": { + "char_loop": 22, + "punctuation_heavy": 22, + "token_3gram_repetition": 22 + }, + "prompts": 30 + }, + "math": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 15, + "eos_hit_rate": 0.5, + "mechanical_bad_rollouts": 27, + "mechanical_bad_rate": 0.9, + "mechanical_flags": { + "char_loop": 1, + "punctuation_heavy": 1, + "token_3gram_repetition": 27 + }, + "prompts": 30 + }, + "reasoning": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 23, + "eos_hit_rate": 0.7666666666666667, + "mechanical_bad_rollouts": 16, + "mechanical_bad_rate": 0.5333333333333333, + "mechanical_flags": { + "char_loop": 1, + "punctuation_heavy": 1, + "token_3gram_repetition": 16 + }, + "prompts": 30 + } + } + }, + "decode_control_rep108": { + "label": "90-prompt greedy decode control: rep108", + "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_decode_sentinel/rep108/passk_summary.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "settings": { + "eos_logit_bias": 0.0, + "include_greedy_rollout": true, + "max_new_tokens": 64, + "no_repeat_ngram_size": 0, + "precision": "fp16", + "prompt_variant": "plain", + "repetition_penalty": 1.08, + "samples_per_prompt": 0, + "seed": 20260610, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.95 + }, + "totals": { + "evaluated_prompts": 90, + "read_prompts": 90, + "rollouts": 90, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 46, + "eos_hit_rate": 0.5111111111111111, + "mechanical_bad_rollouts": 65, + "mechanical_bad_rate": 0.7222222222222222, + "pass_any_categories": [], + "mechanical_flags": { + "char_loop": 24, + "punctuation_heavy": 24, + "token_3gram_repetition": 65 + } + }, + "by_category": { + "instruction": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 8, + "eos_hit_rate": 0.26666666666666666, + "mechanical_bad_rollouts": 22, + "mechanical_bad_rate": 0.7333333333333333, + "mechanical_flags": { + "char_loop": 22, + "punctuation_heavy": 22, + "token_3gram_repetition": 22 + }, + "prompts": 30 + }, + "math": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 15, + "eos_hit_rate": 0.5, + "mechanical_bad_rollouts": 27, + "mechanical_bad_rate": 0.9, + "mechanical_flags": { + "char_loop": 1, + "punctuation_heavy": 1, + "token_3gram_repetition": 27 + }, + "prompts": 30 + }, + "reasoning": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 23, + "eos_hit_rate": 0.7666666666666667, + "mechanical_bad_rollouts": 16, + "mechanical_bad_rate": 0.5333333333333333, + "mechanical_flags": { + "char_loop": 1, + "punctuation_heavy": 1, + "token_3gram_repetition": 16 + }, + "prompts": 30 + } + } + }, + "decode_control_rep108_eos1": { + "label": "90-prompt greedy decode control: rep108_eos1", + "source_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_decode_sentinel/rep108_eos1/passk_summary.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "settings": { + "eos_logit_bias": 1.0, + "include_greedy_rollout": true, + "max_new_tokens": 64, + "no_repeat_ngram_size": 0, + "precision": "fp16", + "prompt_variant": "plain", + "repetition_penalty": 1.08, + "samples_per_prompt": 0, + "seed": 20260610, + "temperature": 0.7, + "top_k": 50, + "top_p": 0.95 + }, + "totals": { + "evaluated_prompts": 90, + "read_prompts": 90, + "rollouts": 90, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 46, + "eos_hit_rate": 0.5111111111111111, + "mechanical_bad_rollouts": 65, + "mechanical_bad_rate": 0.7222222222222222, + "pass_any_categories": [], + "mechanical_flags": { + "char_loop": 24, + "punctuation_heavy": 24, + "token_3gram_repetition": 65 + } + }, + "by_category": { + "instruction": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 8, + "eos_hit_rate": 0.26666666666666666, + "mechanical_bad_rollouts": 22, + "mechanical_bad_rate": 0.7333333333333333, + "mechanical_flags": { + "char_loop": 22, + "punctuation_heavy": 22, + "token_3gram_repetition": 22 + }, + "prompts": 30 + }, + "math": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 15, + "eos_hit_rate": 0.5, + "mechanical_bad_rollouts": 27, + "mechanical_bad_rate": 0.9, + "mechanical_flags": { + "char_loop": 1, + "punctuation_heavy": 1, + "token_3gram_repetition": 27 + }, + "prompts": 30 + }, + "reasoning": { + "rollouts": 30, + "pass_any_prompts": 0, + "pass_any_rate": 0.0, + "pass_rollouts": 0, + "pass_rollout_rate": 0.0, + "greedy_hit_prompts": 0, + "greedy_hit_rate": 0.0, + "sample_hit_prompts": 0, + "sample_hit_rate": 0.0, + "eos_hits": 23, + "eos_hit_rate": 0.7666666666666667, + "mechanical_bad_rollouts": 16, + "mechanical_bad_rate": 0.5333333333333333, + "mechanical_flags": { + "char_loop": 1, + "punctuation_heavy": 1, + "token_3gram_repetition": 16 + }, + "prompts": 30 + } + } + } + }, + "korean_public_proxy_stagegate": { + "belebele_kor": { + "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/belebele_kor.json", + "accuracy": 0.275, + "accuracy_percent": 27.5, + "correct": 55, + "total": 200, + "score": "avg_logprob", + "categories": { + "belebele:kor_Hang": { + "accuracy": 0.275, + "correct": 55, + "total": 200, + "accuracy_percent": 27.5 + } + } + }, + "gsm8k_ko": { + "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/gsm8k_ko.json", + "accuracy": 0.0, + "accuracy_percent": 0.0, + "correct": 0, + "total": 100, + "max_new_tokens": 192 + }, + "haerae_bench_2_0": { + "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/haerae_bench_2_0.json", + "accuracy": 0.16666666666666666, + "accuracy_percent": 16.6667, + "correct": 10, + "total": 60, + "score": "avg_logprob", + "categories": { + "haerae-bench-2.0:context_definition_alignment": { + "accuracy": 0.26666666666666666, + "correct": 8, + "total": 30, + "accuracy_percent": 26.6667 + }, + "haerae-bench-2.0:date_understanding": { + "accuracy": 0.06666666666666667, + "correct": 2, + "total": 30, + "accuracy_percent": 6.6667 + } + } + }, + "hrm8k": { + "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/hrm8k.json", + "accuracy": 0.03333333333333333, + "accuracy_percent": 3.3333, + "correct": 2, + "total": 60, + "max_new_tokens": 192 + }, + "kmmlu_hard": { + "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/kmmlu_hard.json", + "accuracy": 0.2125, + "accuracy_percent": 21.25, + "correct": 34, + "total": 160, + "score": "avg_logprob", + "categories": { + "Biology": { + "accuracy": 0.2, + "correct": 4, + "total": 20, + "accuracy_percent": 20.0 + }, + "Computer Science": { + "accuracy": 0.25, + "correct": 5, + "total": 20, + "accuracy_percent": 25.0 + }, + "Education": { + "accuracy": 0.1, + "correct": 2, + "total": 20, + "accuracy_percent": 10.0 + }, + "Environmental Science": { + "accuracy": 0.2, + "correct": 4, + "total": 20, + "accuracy_percent": 20.0 + }, + "Fashion": { + "accuracy": 0.2, + "correct": 4, + "total": 20, + "accuracy_percent": 20.0 + }, + "Korean History": { + "accuracy": 0.3, + "correct": 6, + "total": 20, + "accuracy_percent": 30.0 + }, + "Psychology": { + "accuracy": 0.2, + "correct": 4, + "total": 20, + "accuracy_percent": 20.0 + }, + "Railway and Automotive Engineering": { + "accuracy": 0.25, + "correct": 5, + "total": 20, + "accuracy_percent": 25.0 + } + } + }, + "kobalt": { + "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/kobalt.json", + "accuracy": 0.095, + "accuracy_percent": 9.5, + "correct": 19, + "total": 200, + "score": "avg_logprob", + "categories": { + "kobalt:Phonetics/Phonology:Phonological Alternation": { + "accuracy": 0.1111111111111111, + "correct": 1, + "total": 9, + "accuracy_percent": 11.1111 + }, + "kobalt:Phonetics/Phonology:Phonological Constraints": { + "accuracy": 0.0, + "correct": 0, + "total": 3, + "accuracy_percent": 0.0 + }, + "kobalt:Pragmatics:Deixis & Reference": { + "accuracy": 0.0, + "correct": 0, + "total": 12, + "accuracy_percent": 0.0 + }, + "kobalt:Pragmatics:Discourse Principles": { + "accuracy": 0.0, + "correct": 0, + "total": 6, + "accuracy_percent": 0.0 + }, + "kobalt:Pragmatics:Implicature": { + "accuracy": 0.14285714285714285, + "correct": 1, + "total": 7, + "accuracy_percent": 14.2857 + }, + "kobalt:Pragmatics:Social Relationship Marking": { + "accuracy": 0.5, + "correct": 1, + "total": 2, + "accuracy_percent": 50.0 + }, + "kobalt:Pragmatics:Speech Acts": { + "accuracy": 0.05555555555555555, + "correct": 1, + "total": 18, + "accuracy_percent": 5.5556 + }, + "kobalt:Semantics:Conjunctions": { + "accuracy": 0.0, + "correct": 0, + "total": 2, + "accuracy_percent": 0.0 + }, + "kobalt:Semantics:Inter-sentence Relations": { + "accuracy": 0.0, + "correct": 0, + "total": 1, + "accuracy_percent": 0.0 + }, + "kobalt:Semantics:Rhetorical Expressions": { + "accuracy": 0.3333333333333333, + "correct": 1, + "total": 3, + "accuracy_percent": 33.3333 + }, + "kobalt:Semantics:Semantic Compatibility": { + "accuracy": 0.14285714285714285, + "correct": 1, + "total": 7, + "accuracy_percent": 14.2857 + }, + "kobalt:Semantics:Word Relationships": { + "accuracy": 0.13636363636363635, + "correct": 3, + "total": 22, + "accuracy_percent": 13.6364 + }, + "kobalt:Syntax:Agreement": { + "accuracy": 0.09803921568627451, + "correct": 5, + "total": 51, + "accuracy_percent": 9.8039 + }, + "kobalt:Syntax:Argument Structure & Valency": { + "accuracy": 0.12121212121212122, + "correct": 4, + "total": 33, + "accuracy_percent": 12.1212 + }, + "kobalt:Syntax:Embedded Clauses": { + "accuracy": 0.041666666666666664, + "correct": 1, + "total": 24, + "accuracy_percent": 4.1667 + } + } + }, + "kopiqa": { + "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/kopiqa.json", + "accuracy": 0.49, + "accuracy_percent": 49.0, + "correct": 98, + "total": 200, + "score": "avg_logprob", + "categories": { + "ko-piqa:1.0": { + "accuracy": 0.5681818181818182, + "correct": 25, + "total": 44, + "accuracy_percent": 56.8182 + }, + "ko-piqa:None": { + "accuracy": 0.46794871794871795, + "correct": 73, + "total": 156, + "accuracy_percent": 46.7949 + } + } + }, + "korean_mini_mcq": { + "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/korean_mini_mcq.json", + "accuracy": 0.6, + "accuracy_percent": 60.0, + "correct": 12, + "total": 20, + "score": "avg_logprob", + "categories": { + "commonsense": { + "accuracy": 0.3333333333333333, + "correct": 1, + "total": 3, + "accuracy_percent": 33.3333 + }, + "culture": { + "accuracy": 1.0, + "correct": 1, + "total": 1, + "accuracy_percent": 100.0 + }, + "instruction": { + "accuracy": 1.0, + "correct": 3, + "total": 3, + "accuracy_percent": 100.0 + }, + "logic": { + "accuracy": 0.6666666666666666, + "correct": 2, + "total": 3, + "accuracy_percent": 66.6667 + }, + "math": { + "accuracy": 0.3333333333333333, + "correct": 1, + "total": 3, + "accuracy_percent": 33.3333 + }, + "reading": { + "accuracy": 0.6666666666666666, + "correct": 2, + "total": 3, + "accuracy_percent": 66.6667 + }, + "safety": { + "accuracy": 0.5, + "correct": 1, + "total": 2, + "accuracy_percent": 50.0 + }, + "summary": { + "accuracy": 0.5, + "correct": 1, + "total": 2, + "accuracy_percent": 50.0 + } + } + }, + "winogrande_ko": { + "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_stagegate_presentinel_20260609/winogrande_ko.json", + "accuracy": 0.455, + "accuracy_percent": 45.5, + "correct": 91, + "total": 200, + "score": "avg_logprob", + "categories": { + "winogrande-ko": { + "accuracy": 0.455, + "correct": 91, + "total": 200, + "accuracy_percent": 45.5 + } + } + } + }, + "boundary_and_forensic_diagnostics": { + "passk32_argmax_boundary_pairs": { + "source_json": "/tmp/chojm_hw2_runs/argmax_boundary_eval_20260609/clean7020_passk32/metrics.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "settings": { + "pairs": 418, + "top_k": 16, + "margin_threshold": 0.55, + "with_rank": true + }, + "overall": { + "count": 418, + "target_top1": 0, + "target_top1_rate": 0.0, + "target_topk": 327, + "target_topk_rate": 0.7822966507177034, + "target_gt_negative": 0, + "target_gt_negative_rate": 0.0, + "target_margin_ge_0": 1, + "target_margin_ge_0_rate": 0.0023923444976076554, + "target_margin_ge_threshold": 0, + "target_margin_ge_threshold_rate": 0.0, + "margin_mean": -2.341340279465087, + "target_logit_mean": 7.655613219338741, + "negative_logit_mean": 9.996953498803828, + "target_rank_mean": 516.4282296650717 + }, + "by_task": [ + { + "task_type": "clean7020_passk32_inventory_accepted_only_instruction", + "count": 314, + "target_top1": 0, + "target_top1_rate": 0.0, + "target_topk": 244, + "target_topk_rate": 0.7770700636942676, + "target_gt_negative": 0, + "target_gt_negative_rate": 0.0, + "target_margin_ge_0": 1, + "target_margin_ge_0_rate": 0.0031847133757961785, + "target_margin_ge_threshold": 0, + "target_margin_ge_threshold_rate": 0.0, + "margin_mean": -2.4404563174885547, + "target_logit_mean": 7.31813992056877, + "negative_logit_mean": 9.758596238057326, + "target_rank_mean": 409.171974522293 + }, + { + "task_type": "clean7020_passk32_inventory_accepted_only_math", + "count": 69, + "target_top1": 0, + "target_top1_rate": 0.0, + "target_topk": 60, + "target_topk_rate": 0.8695652173913043, + "target_gt_negative": 0, + "target_gt_negative_rate": 0.0, + "target_margin_ge_0": 0, + "target_margin_ge_0_rate": 0.0, + "target_margin_ge_threshold": 0, + "target_margin_ge_threshold_rate": 0.0, + "margin_mean": -1.7986016757246377, + "target_logit_mean": 9.565755208333334, + "negative_logit_mean": 11.36435688405797, + "target_rank_mean": 26.942028985507246 + }, + { + "task_type": "clean7020_passk32_inventory_accepted_only_reasoning", + "count": 35, + "target_top1": 0, + "target_top1_rate": 0.0, + "target_topk": 23, + "target_topk_rate": 0.6571428571428571, + "target_gt_negative": 0, + "target_gt_negative_rate": 0.0, + "target_margin_ge_0": 0, + "target_margin_ge_0_rate": 0.0, + "target_margin_ge_threshold": 0, + "target_margin_ge_threshold_rate": 0.0, + "margin_mean": -2.5220982142857142, + "target_logit_mean": 6.917522321428572, + "negative_logit_mean": 9.439620535714285, + "target_rank_mean": 2443.657142857143 + } + ] + }, + "gspo_scoreseq_parent_boundary_pairs": { + "source_json": "/tmp/chojm_hw2_runs/eval/clean7020_gspo_scoreseq_v1_boundary_parent7020.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "settings": { + "pairs": 33226, + "top_k": 5, + "margin_threshold": 0.65, + "with_rank": true + }, + "overall": { + "count": 33226, + "target_top1": 588, + "target_top1_rate": 0.017696984289411907, + "target_topk": 7844, + "target_topk_rate": 0.23608017817371937, + "target_gt_negative": 662, + "target_gt_negative_rate": 0.019924155781616805, + "target_margin_ge_0": 679, + "target_margin_ge_0_rate": 0.020435803286582798, + "target_margin_ge_threshold": 70, + "target_margin_ge_threshold_rate": 0.002106783843977608, + "margin_mean": -4.19784176139393, + "target_logit_mean": 4.610714955296313, + "negative_logit_mean": 8.808556716690243, + "target_rank_mean": 1509.9462168181544 + } + }, + "trainonly_grounded_bridge_forensics_shard00": { + "source_json": "/tmp/chojm_hw2_runs/eval/trainonly_grounded_bridge6990_vs_clean7020_20260609/clean7020_shard00/generation-forensics.json", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "summary": { + "greedy|plain": { + "rows": 128, + "bad": 128, + "bad_rate": 1.0, + "eos_hit": 36, + "eos_hit_rate": 0.28125, + "semantic_hit": 0, + "semantic_hit_rate": 0.0, + "avg_forensic_score": 31.9921875, + "flags": { + "semantic_weak_miss": 128, + "max_new_tokens_hit": 92, + "token_3gram_repetition": 99, + "char_loop": 37, + "punctuation_heavy": 38, + "low_hangul_alnum": 35, + "role_leak": 1 + } + } + } + } + }, + "public_comparison_and_replacement_references": { + "benchmark_results_file": "benchmark_results_20260609.json", + "replacement_evidence_file": "replacement_evidence_20260609.json", + "benchmark_results_summary": { + "metric": "Korean semantic pass@33", + "prompt_set": "/tmp/chojm_hw2_runs/data/clean7020_passk32_accepted_imr_20260609/sentinel_imr_30x3_90.jsonl", + "decoding": { + "include_greedy_rollout": true, + "samples_per_prompt": 32, + "temperature": 0.7, + "top_p": 0.95, + "top_k": 50, + "max_new_tokens": 64 + }, + "models": { + "Jumini-Ko-1.2B-clean7020": { + "params": 1260505088, + "language_focus": "Korean from-scratch", + "summary_json": "/tmp/chojm_hw2_runs/passk_basin_inventory_20260609/clean7020_imr_k32_gpu2/passk_summary.json", + "pass_any": "38/90", + "pass_any_score": 42.22222222222222, + "pass_rollouts": "131/2884", + "pass_rollout_score": 4.542302357836339, + "greedy_hit": "0/90", + "categories": { + "instruction": "26/30", + "math": "8/30", + "reasoning": "4/30" + } + }, + "TinyLlama-1.1B-intermediate-step-1431k-3T": { + "repo_id": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T", + "params": 1100000000, + "language_focus": "English general base", + "summary_json": "/tmp/chojm_hw2_runs/hf_baseline_compare_20260609/tinyllama_1p1b_base_k32/summary.json", + "pass_any": "31/90", + "pass_any_score": 34.44444444444444, + "pass_rollouts": "56/2966", + "pass_rollout_score": 1.8880647336480108, + "greedy_hit": "0/90", + "categories": { + "instruction": "27/30", + "math": "4/30", + "reasoning": "0/30" + } + } + } + }, + "replacement_summary": { + "name": "clean7020", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "export_dir": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32", + "strat240_pass_any": "21/240", + "strat240_pass_rollouts": "29/2105", + "strat240_mechanical_bad_rollouts": "856/2105", + "k32_pass_any": "38/90", + "strict_heldout_stable_semantic_hits": "2/12" + }, + "previous_hf_reference": { + "name": "v3a6200", + "remote_sha_before_replace": "369600b6f14950e4bf6c7c43457ed3ee0921663c", + "checkpoint": "/raid2/chojm/hw2_llm_from_scratch/experiments/jumini-ko-1.2b-phase2-mixture-v3a-retention-continue-5700-to-6200-2gpu12-mb9/step_00006200/model.pt", + "strat240_pass_any": "0/240", + "strat240_pass_rollouts": "0/2094", + "strat240_mechanical_bad_rollouts": "1755/2094" + } + }, + "omission_policy": [ + "No row-level prompt text or completions are included in this public score appendix.", + "Non-exported descendants, row-edit probes, merge/soup probes, and alternate 7020-lineage diagnostics are not presented as scores for the current HF checkpoint.", + "External baseline comparison scores remain in benchmark_results_20260609.json and are not duplicated here except by reference." + ] +} diff --git a/load_smoke.json b/load_smoke.json new file mode 100644 index 0000000..5794321 --- /dev/null +++ b/load_smoke.json @@ -0,0 +1,19 @@ +{ + "model_dir": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32", + "load_model": true, + "model_type": "llama", + "architectures": [ + "LlamaForCausalLM" + ], + "vocab_size": 128000, + "hidden_size": 2048, + "num_hidden_layers": 28, + "tokenizer_len": 128000, + "bos_token": "[BOS]", + "eos_token": "[EOS]", + "pad_token": "[PAD]", + "model_class": "LlamaForCausalLM", + "params": 1260505088, + "parameter_count": 1260505088, + "ok": true +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..852b277 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e355c9f2c5e6f8f5c8b9074c5ea944838eb9728b86c7abd7ac26c460fcd3bf1 +size 2521039488 diff --git a/replacement_evidence_20260609.json b/replacement_evidence_20260609.json new file mode 100644 index 0000000..dbe7f63 --- /dev/null +++ b/replacement_evidence_20260609.json @@ -0,0 +1,32 @@ +{ + "repo_id": "properly59/Jumini-Ko-1.2B", + "replacement_date": "2026-06-09", + "previous_hf_checkpoint": { + "name": "v3a6200", + "remote_sha_before_replace": "369600b6f14950e4bf6c7c43457ed3ee0921663c", + "checkpoint": "/raid2/chojm/hw2_llm_from_scratch/experiments/jumini-ko-1.2b-phase2-mixture-v3a-retention-continue-5700-to-6200-2gpu12-mb9/step_00006200/model.pt", + "strat240_pass_any": "0/240", + "strat240_pass_rollouts": "0/2094", + "strat240_mechanical_bad_rollouts": "1755/2094" + }, + "new_checkpoint": { + "name": "clean7020", + "checkpoint": "/tmp/chojm_hw2_runs/jumini-ko-1.2b-firsttok7000-argmax-boundary-layernormshock-v1-last16-4gpu0123-mb8-probe25-rerun20260608/step_00007020/model.pt", + "export_dir": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32", + "strat240_pass_any": "21/240", + "strat240_pass_rollouts": "29/2105", + "strat240_mechanical_bad_rollouts": "856/2105", + "k32_pass_any": "38/90", + "strict_heldout_stable_semantic_hits": "2/12" + }, + "validation": { + "load_smoke": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32/load_smoke.json", + "hf_equivalence": "models/jumini-ko-1.2b-clean7020-best-20260609-hf-fp32/hf_equivalence_atol2e-5.json", + "hf_equivalence_argmax_mismatch_rate": 0.0, + "hf_equivalence_max_abs_diff": 0.000014781951904296875 + }, + "notes": [ + "This is a pragmatic replacement over the existing HF upload because local average diagnostics are better than v3a6200.", + "Strict heldout remains unsolved; this is not labeled as a final strong-candidate breakthrough." + ] +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..529ffeb --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,6 @@ +{ + "bos_token": "[BOS]", + "eos_token": "[EOS]", + "unk_token": "[UNK]", + "pad_token": "[PAD]" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..e01cc2e --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02a1d2e971468994add44673bd84e6ca77735752515a402e196d12336d1283ac +size 11624392 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..f6ef5ba --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,12 @@ +{ + "backend": "tokenizers", + "bos_token": "[BOS]", + "eos_token": "[EOS]", + "fix_mistral_regex": true, + "is_local": true, + "local_files_only": false, + "model_max_length": 4096, + "pad_token": "[PAD]", + "tokenizer_class": "TokenizersBackend", + "unk_token": "[UNK]" +}