From 56add3de1929506dae2e06a8043676c5ac46a2e4 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 25 Sep 2026 06:07:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: idealab-cs2/zagreus-0.4B-italic-softkd Source: Original Platform --- .gitattributes | 36 ++++++++++++++++++++++ README.md | 52 ++++++++++++++++++++++++++++++++ chat_template.jinja | 68 ++++++++++++++++++++++++++++++++++++++++++ config.json | 32 ++++++++++++++++++++ generation_config.json | 6 ++++ model.safetensors | 3 ++ tokenizer.json | 3 ++ tokenizer_config.json | 16 ++++++++++ 8 files changed, 216 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..7243264 --- /dev/null +++ b/README.md @@ -0,0 +1,52 @@ +--- +license: apache-2.0 +language: +- it +base_model: mii-llm/zagreus-0.4B-ita +pipeline_tag: text-generation +datasets: +- idealab-cs2/italic-softkd-pool +tags: +- italian +- small-language-model +- knowledge-distillation +- italic +--- + +# zagreus-0.4B-italic-softkd + +Submission for the [mii-llm Italian Post-Training Challenge](https://huggingface.co/spaces/mii-llm/Post-Training-Challenge). This is [mii-llm/zagreus-0.4B-ita](https://huggingface.co/mii-llm/zagreus-0.4B-ita) (437M, Llama architecture) post-trained for the [ITALIC benchmark](https://github.com/Crisp-Unimib/ITALIC), 10,000 multiple-choice questions on Italian culture and language. + +| Model | ITALIC full 10K (5-shot fast, temp 0, official harness) | +|---|---| +| zagreus-0.4B-ita (base) | 0.2802 | +| public leaderboard best at submission time | 0.372 | +| **this model** | **0.4787** (confirmed 0.4784 on an independent rerun) | + +**Update (2026-08-05):** superseded by [idealab-cs2/zagreus-0.4B-italic-extkd](https://huggingface.co/idealab-cs2/zagreus-0.4B-italic-extkd) — same recipe extended with two more softKD stages (culture pool, then external in-distribution pool), 0.4921/0.4929/0.4932 full-10K. + +## Training + +One soft knowledge-distillation run from the base model, within the challenge budget of 10 H100 GPU hours (measured replay: ~4.9 H100h including data generation, teacher labeling and evaluation). + +For every training question a 3-teacher committee (Mistral-Small-3.2-24B, Gemma-3-27B, Qwen3-32B, each fitting a single H100 in bf16) provides its probability distribution over the answer letters at the answer token. The student is trained in fp32 with a KL term on that distribution (temperature 2.0) plus cross-entropy on the gold letter (weight 0.5). The training pool has 21,606 items: pinocchio-derived knowledge MCQs, committee-labeled grammar MCQs, and generated orthography/morphology MCQs for the categories where public data is scarce. Every pool is decontaminated against the ITALIC test set (exact match plus semantic similarity at cosine 0.80; 0 exact leaks, max similarity 0.7999). + +Training data is published on the Hub: [italic-softkd-pool](https://huggingface.co/datasets/idealab-cs2/italic-softkd-pool) (the exact training file) (the exact training file; provenance splits knowledge/grammar/orthomorph included as named splits). Training code, the decontamination manifest and the complete report (including what did not work) are in the submission repository: https://github.com/ruggsea/italic-challenge-submission + +## Evaluation + +Official [ITALIC harness](https://github.com/Crisp-Unimib/ITALIC), unmodified: vLLM server, fast mode (letter-only answers), 5-shot, temperature 0, accuracy over all 10,000 questions. The chat template is embedded in the tokenizer config; serve the model with its own template. + +```bash +vllm serve idealab-cs2/zagreus-0.4B-italic-softkd --chat-template-content-format auto +``` + +Per-category accuracy: civic education .56, lexicon .52, tourism .50, geography .47, art history .45, history .45, synonyms .47, syntax .42, orthography .39, morphology .36. + +## Acknowledgments + +Training, data curation and evaluation by [ruggsea](https://huggingface.co/ruggsea), using compute resources of the Complex Social & Computational Systems (CS²) group, IDea_Lab, University of Graz. + +## Intended use + +This model is specialized for Italian multiple-choice question answering in the ITALIC format. It is a 0.4B model: outside this format expect base-model-level capabilities. diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..e129b5d --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,68 @@ +{{ bos_token }} +{% if tools %} +<|im_start|>system +{% if messages and messages[0]['role'] == 'system' %} +{{ messages[0]['content'] }} +{% endif %} +# Tools +You may call one or more functions to assist with the user query. +You are provided with function signatures within XML tags: + +{% for tool in tools %} +{{ tool | tojson }} +{% endfor %} + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } +<|im_end|> +{% else %} +{% if messages and messages[0]['role'] == 'system' %} +<|im_start|>system +{{ messages[0]['content'] }}<|im_end|> +{% endif %} +{% endif %} + +{% for message in messages %} +{% if message.content is string %} + {% set content = message.content %} +{% else %} + {% set content = '' %} +{% endif %} + +{% if (message.role == 'user') or (message.role == 'system' and not loop.first) %} +<|im_start|>{{ message.role }} +{{ content }}<|im_end|> + +{% elif message.role == 'assistant' %} +<|im_start|>assistant +{{ content }} +{% if message.tool_calls %} + {% for tool_call in message.tool_calls %} + {% if tool_call.function %} + {% set tool_call = tool_call.function %} + {% endif %} + +{"name": "{{ tool_call.name }}", "arguments": {% if tool_call.arguments is string %}{{ tool_call.arguments }}{% else %}{{ tool_call.arguments | tojson }}{% endif %}} + + {% endfor %} +{% endif %} +<|im_end|> + +{% elif message.role == 'tool' %} +{% if loop.first or (messages[loop.index0 - 1].role != 'tool') %} +<|im_start|>user +{% endif %} + +{{ content }} + +{% if loop.last or (messages[loop.index0 + 1].role != 'tool') %} +<|im_end|> +{% endif %} + +{% endif %} +{% endfor %} + +{% if add_generation_prompt %} +<|im_start|>assistant +{% endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..4ef96b8 --- /dev/null +++ b/config.json @@ -0,0 +1,32 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "float32", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 960, + "initializer_range": 0.02, + "intermediate_size": 2560, + "max_position_embeddings": 4096, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 15, + "num_hidden_layers": 32, + "num_key_value_heads": 5, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "default" + }, + "tie_word_embeddings": true, + "transformers_version": "5.14.1", + "use_cache": true, + "vocab_size": 128262 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..96f05b0 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": 128001, + "transformers_version": "5.14.1" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..48f001f --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff8a67f09e1801ceabe4981173d0b0a8511edee242e46be8f147a1b0685c4097 +size 1751099616 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..980d1f7 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a80502037c38583156839ad1486db1e7e5aa0b3f9d47e072f16e6b30a0eb2dd +size 17211058 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..0f80544 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,16 @@ +{ + "backend": "tokenizers", + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|im_end|>", + "is_local": true, + "local_files_only": false, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "TokenizersBackend", + "chat_template": "{{ bos_token }}\n{% if tools %}\n<|im_start|>system\n{% if messages and messages[0]['role'] == 'system' %}\n{{ messages[0]['content'] }}\n{% endif %}\n# Tools\nYou may call one or more functions to assist with the user query.\nYou are provided with function signatures within XML tags:\n\n{% for tool in tools %}\n{{ tool | tojson }}\n{% endfor %}\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n{% else %}\n{% if messages and messages[0]['role'] == 'system' %}\n<|im_start|>system\n{{ messages[0]['content'] }}<|im_end|>\n{% endif %}\n{% endif %}\n\n{% for message in messages %}\n{% if message.content is string %}\n {% set content = message.content %}\n{% else %}\n {% set content = '' %}\n{% endif %}\n\n{% if (message.role == 'user') or (message.role == 'system' and not loop.first) %}\n<|im_start|>{{ message.role }}\n{{ content }}<|im_end|>\n\n{% elif message.role == 'assistant' %}\n<|im_start|>assistant\n{{ content }}\n{% if message.tool_calls %}\n {% for tool_call in message.tool_calls %}\n {% if tool_call.function %}\n {% set tool_call = tool_call.function %}\n {% endif %}\n\n{\"name\": \"{{ tool_call.name }}\", \"arguments\": {% if tool_call.arguments is string %}{{ tool_call.arguments }}{% else %}{{ tool_call.arguments | tojson }}{% endif %}}\n\n {% endfor %}\n{% endif %}\n<|im_end|>\n\n{% elif message.role == 'tool' %}\n{% if loop.first or (messages[loop.index0 - 1].role != 'tool') %}\n<|im_start|>user\n{% endif %}\n\n{{ content }}\n\n{% if loop.last or (messages[loop.index0 + 1].role != 'tool') %}\n<|im_end|>\n{% endif %}\n\n{% endif %}\n{% endfor %}\n\n{% if add_generation_prompt %}\n<|im_start|>assistant\n{% endif %}\n" +} \ No newline at end of file