From 292111100a1ada923d35644714378d910783fc1d Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 10 Jul 2026 01:10:11 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: EphAsad/Atem-1.7B Source: Original Platform --- .gitattributes | 43 +++++ Atem-1.7B.Q4_K_M.gguf | 3 + Atem-1.7B.Q5_K_M.gguf | 3 + Atem-1.7B.Q8_0.gguf | 3 + Logo.png | 3 + Modelfile | 59 +++++++ README.md | 378 +++++++++++++++++++++++++++++++++++++++++ chat_template.jinja | 100 +++++++++++ config.json | 64 +++++++ generation_config.json | 13 ++ model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 234 +++++++++++++++++++++++++ 13 files changed, 909 insertions(+) create mode 100644 .gitattributes create mode 100644 Atem-1.7B.Q4_K_M.gguf create mode 100644 Atem-1.7B.Q5_K_M.gguf create mode 100644 Atem-1.7B.Q8_0.gguf create mode 100644 Logo.png create mode 100644 Modelfile create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0fbbad9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,43 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +Qwen3-1.7B.Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +Qwen3-1.7B.Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Qwen3-1.7B.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Atem-1.7B.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Atem-1.7B.Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text +Atem-1.7B.Q8_0.gguf filter=lfs diff=lfs merge=lfs -text +Logo.png filter=lfs diff=lfs merge=lfs -text diff --git a/Atem-1.7B.Q4_K_M.gguf b/Atem-1.7B.Q4_K_M.gguf new file mode 100644 index 0000000..2177a98 --- /dev/null +++ b/Atem-1.7B.Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:468e13062fa69b650c781ee30f037993175671be246b2069d1d3777b59f74109 +size 1107409664 diff --git a/Atem-1.7B.Q5_K_M.gguf b/Atem-1.7B.Q5_K_M.gguf new file mode 100644 index 0000000..f06a67b --- /dev/null +++ b/Atem-1.7B.Q5_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:854a78deb78266a9f1f59250fe3c5a283d333be7699e7d6bafae3d9317061a13 +size 1257880320 diff --git a/Atem-1.7B.Q8_0.gguf b/Atem-1.7B.Q8_0.gguf new file mode 100644 index 0000000..ee89290 --- /dev/null +++ b/Atem-1.7B.Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1464b1b04ed8a2bc460c5a35984de0014350ebc1d9663c1d0f793cb2fa831905 +size 1834427136 diff --git a/Logo.png b/Logo.png new file mode 100644 index 0000000..2f31b1d --- /dev/null +++ b/Logo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:036d268ac79d1a5355a5ea602fc02ba312f4938c3506ddfa148ec7de44b69607 +size 981796 diff --git a/Modelfile b/Modelfile new file mode 100644 index 0000000..6007d28 --- /dev/null +++ b/Modelfile @@ -0,0 +1,59 @@ + +FROM Atem-1.7B.Q8_0.gguf +TEMPLATE """{{- if .Messages }} +{{- if or .System .Tools }}<|im_start|>system +{{- if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end }}<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ end }} +{{- end }} +{{- else }} +{{- if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }}""" +PARAMETER stop "<|im_end|>" +PARAMETER stop "<|im_start|>" +PARAMETER temperature 0.6 +PARAMETER min_p 0.0 +PARAMETER top_k 20 +PARAMETER top_p 0.95 +PARAMETER repeat_penalty 1 \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..1ca88be --- /dev/null +++ b/README.md @@ -0,0 +1,378 @@ +--- +license: apache-2.0 +base_model: Qwen/Qwen3-1.7B +tags: +- unsloth +- lora +- qwen3 +- reasoning +- distillation +- chain-of-thought +datasets: +- mitroitskii/OpenR1-Math-220k-formatted +- Jackrong/Claude-opus-4.6-TraceInversion-9000x +- Jackrong/Kimi-K2.5-Reasoning-1M-Cleaned +- WithinUsAI/MiniMax_M2.7_Distilled_5k +- FreedomIntelligence/medical-o1-reasoning-SFT +- Modotte/CodeX-2M-Thinking +- trjxter/DeepSeek-V4-Pro-Reasoning-8000x +- nvidia/OpenCodeReasoning +- openai/gsm8k +language: +- en +pipeline_tag: text-generation +library_name: transformers +--- + +![Atem Logo](https://huggingface.co/EphAsad/Atem-1.7B/resolve/main/Logo.png) + +# Atem-1.7B + +*Ancient logic. Modern intelligence.* + +A 1.7B reasoning model trained via a single CoT-preserving SFT pass directly on Qwen3-1.7B, distilling multi-domain reasoning capability from frontier teacher models while keeping the base model's native thinking capability intact. + +![Base Model](https://img.shields.io/badge/Base-Qwen3--1.7B-blue)![Method](https://img.shields.io/badge/Method-CoT--Preserving%20SFT-purple)![Parameters](https://img.shields.io/badge/Parameters-1.7B-orange)![License](https://img.shields.io/badge/License-Apache%202.0-green) + +--- + +## Overview + +Atem-1.7B is a 1.7B parameter reasoning model built via a single supervised fine-tuning pass on raw Qwen3-1.7B, using the same CoT-preserving single-pass design as Atem-4B and Atem-8B. It is the most compute-efficient model in the Atem series, completing training in under 2.5 hours on an A100-SXM4 80GB while maintaining 2.95% proportional LoRA capacity — close to the series-wide 3% target. + +This model includes GSM8K-format training examples (5K no-think records) to partially restore the `####` answer convention that the reasoning corpus otherwise overwrites — an improvement over Atem-4B and Atem-8B, which did not include these. + +--- + +## Model Details + +| Property | Value | +| --- | --- | +| **Base model** | Qwen/Qwen3-1.7B | +| **Training method** | Single-pass CoT-Preserving LoRA SFT | +| **LoRA config** | r=48, alpha=96, dropout=0.05 | +| **Target modules** | q, k, v, o, gate, up, down projections | +| **Parameters** | ~1.77B | +| **Trainable (LoRA) params** | 52,297,728 (2.95% of base) | +| **Training records** | 62,301 (after token-length filtering) | +| **Think / No-think split** | 85% / 15% | +| **Epochs** | 2 (ceiling; early stopping patience=3, never triggered) | +| **Effective batch size** | 64 (batch 16 × grad accum 4) | +| **Learning rate** | 1e-4, cosine schedule, 5% warmup | +| **Max sequence length** | 6,144 tokens | +| **Precision** | bfloat16 (full 16-bit LoRA, not QLoRA) | +| **Hardware** | NVIDIA A100-SXM4 80GB | +| **Runtime** | 2h28m | +| **License** | Apache 2.0 | + +--- + +## Design Notes + +**Single combined pass.** The same single CoT-preserving pass design used across Atem-4B and Atem-8B — no erase-then-rebuild pipeline. Reasoning capability is built directly on the base model's intact native foundation. + +**r=48 for proportional capacity.** r=32 on a 1.7B model represents only 2.05% of the model's parameters — the same shrinking-fraction problem observed across the series as model size grows. r=48 recovers 2.95% proportional capacity, close to the series-wide ~3% target and significantly better than r=32 would have provided. + +**GSM8K format restoration.** The standard Atem training corpus uses `\boxed{}` notation throughout. Atem-4B and Atem-8B both showed a systematic GSM8K strict-match regression as a result of this format shift. Atem-1.7B is the first in the series to include 5,000 GSM8K-format training examples (from `openai/gsm8k`) in the no-think pool, partially re-establishing the `#### answer` convention alongside `\boxed{}`. + +**Full 16-bit LoRA.** At 1.7B the model weights occupy only ~3.4GB, leaving over 75GB of A100 headroom. Full 16-bit LoRA is used throughout — faster and marginally more accurate than QLoRA without any VRAM constraint. + +--- + +## Intended Use + +Atem-1.7B is suited for reasoning tasks on resource-constrained hardware — edge devices, local deployment, and applications where a 4B+ model is impractical: + +- Multi-step mathematical reasoning +- Code explanation, implementation, and debugging +- Analytical reasoning across diverse domains +- Commonsense reasoning and physical intuition +- Logic and argument evaluation + +For higher capability at the cost of resource requirements, Atem-4B and Atem-8B provide progressively stronger results on the same reasoning tasks. + +--- + +## Training Data + +Atem-1.7B was trained on the same eight-source reasoning corpus as Atem-4B and Atem-8B, with the addition of 5,000 GSM8K-format records to partially restore the `####` answer convention. All sources include explicit chain-of-thought reasoning traces; 85% of training records were formatted with full think traces and 15% as direct answers. + +| Dataset | Records | Source / Teacher | +| --- | --- | --- | +| mitroitskii/OpenR1-Math-220k-formatted | ~10,938 | DeepSeek-R1 — Mathematics (correctness-filtered) | +| Jackrong/Claude-opus-4.6-TraceInversion-9000x | 7,000 | Claude Opus 4.6 — Trace Inversion | +| Jackrong/Kimi-K2.5-Reasoning-1M-Cleaned (General-Math) | 8,000 | Kimi K2.5 — Mathematical Reasoning | +| Jackrong/Kimi-K2.5-Reasoning-1M-Cleaned (General-Distillation) | 8,000 | Kimi K2.5 — General Reasoning | +| Jackrong/Kimi-K2.5-Reasoning-1M-Cleaned (PHD-Science) | 8,000 | Kimi K2.5 — Scientific Reasoning | +| WithinUsAI/MiniMax_M2.7_Distilled_5k | 5,000 | MiniMax M2.7 | +| FreedomIntelligence/medical-o1-reasoning-SFT | 7,500 | Medical reasoning (English config) | +| Modotte/CodeX-2M-Thinking | 15,000 | Mixed — Coding with CoT | +| trjxter/DeepSeek-V4-Pro-Reasoning-8000x | ~8,014 | DeepSeek-V4-Pro | +| nvidia/OpenCodeReasoning | 15,000 | Mixed — Competitive coding | +| openai/gsm8k (no-think) | 5,000 | GSM8K `#### answer` format restoration | +| **Total (pre-filter pool)** | **96,017** | | +| **Total (post-filter, trained on)** | **62,301** | | + +Non-English reasoning traces (primarily CJK) were filtered at the trace level using an ASCII-ratio threshold and retained as no-think records. The 34.3% filter rate is consistent with Atem-4B (32.7%) and Atem-8B (34.3%) at the same 6,144-token ceiling. + +--- + +## Training Configuration + +```python +# Key hyperparameters +lora_r = 48 +lora_alpha = 96 +lora_dropout = 0.05 +max_seq_length = 6144 +learning_rate = 1e-4 +lr_scheduler = 'cosine' +warmup_ratio = 0.05 +batch_size = 16 +grad_accumulation = 4 # effective batch size: 64 +num_epochs = 2 # ceiling — early stopping patience=3 +eval_steps = 150 +early_stopping_patience = 3 +early_stopping_threshold = 0.001 +nothink_ratio = 0.15 +load_in_4bit = False # full 16-bit LoRA +dtype = bfloat16 +``` + +--- + +## Loss Curve + +| Step | Train Loss | Val Loss | +| --- | --- | --- | +| 150 | 1.0706 | 1.0833 | +| 300 | 1.0385 | 1.0520 | +| 450 | 1.0566 | 1.0372 | +| 600 | 0.9990 | 1.0255 | +| 750 | 1.0082 | 1.0158 | +| 900 | 0.9887 | 1.0091 | +| 1050 | 0.9294 | 1.0051 | +| 1200 | 0.8906 | 1.0020 | +| 1350 | 0.9331 | 0.9993 | +| 1500 | 0.9780 | 0.9973 | +| 1650 | 0.9467 | 0.9963 | +| 1800 | 0.9341 | 0.9957 | +| Final (1948) | **0.9902** (avg) | **0.9956** | + +Train loss is noisier than in larger Atem models — characteristic of smaller models with a diverse multi-domain corpus. Validation loss improved monotonically across all 13 checkpoints without exception. Early stopping was configured but never triggered. + +--- + +## Evaluation + +### Benchmark Results + +Evaluated against base Qwen3-1.7B (`Qwen/Qwen3-1.7B`) using lm-evaluation-harness. Both models were loaded in 4-bit for evaluation. Statistical significance (σ) is provided as context for interpreting each result — at 1.7B scale, several deltas that appear directionally positive are within sampling noise due to test set size. + +| Task | Base (Qwen3-1.7B) | Atem-1.7B | Delta | σ | +| --- | --- | --- | --- | --- | +| ARC-Challenge (0-shot, acc_norm) | 40.7% | 42.2% | +1.5pp ✓ | 0.7σ | +| GSM8K strict (5-shot, exact_match) | 62.0% | 58.7% | −3.3pp ⚠ | 1.7σ | +| HellaSwag (0-shot, acc_norm) | 59.4% | **61.3%** | **+1.9pp** ✓ | 2.8σ | +| MMLU (0-shot, acc) | 55.4% | 56.2% | +0.8pp ✓ | 1.3σ | +| Winogrande (0-shot, acc) | 61.8% | 61.1% | −0.7pp ⚠ | 0.4σ | +| PIQA (0-shot, acc) | 71.4% | 71.4% | +0.0pp — | 0.0σ | +| OpenBookQA (0-shot, acc_norm) | 36.0% | **39.0%** | +3.0pp ✓ | 1.0σ | +| BoolQ (0-shot, acc) | 76.5% | 76.0% | −0.5pp — | 0.5σ | + +**HellaSwag (+1.9pp, 2.8σ)** is the only clearly statistically significant positive result. It uses normalised log-likelihood scoring over multiple-choice options — format-independent and not influenced by generation style. This is also the most consistent signal across the full Atem series (1.7B: +1.9pp, 4B: +2.9pp, 8B: +1.7pp), confirming genuine commonsense reasoning transfer from the CoT training corpus. + +**OpenBookQA (+3.0pp)** is directionally strong but the test set is only 500 questions, giving 1.0σ — treat this as encouraging rather than conclusive. + +**Winogrande (−0.7pp, ⚠)** despite the flag is 0.4σ and statistically indistinguishable from noise. Not a meaningful regression. + +**MMLU (+0.8pp, 1.3σ)** is borderline. Consistent with the series pattern — neither model has a knowledge breadth advantage after CoT training. + +Results at 1.7B are generally less pronounced than at 4B and 8B, as expected: smaller models with proportionally larger parameter changes per training step exhibit noisier benchmark behaviour, and the absolute capability headroom above random baselines is narrower. + +### GSM8K — Formatting Shift + +The strict-match regression (−3.3pp) follows the same pattern established at 4B and 8B: the training corpus uses `\boxed{}` notation, systematically shifting away from the `####` format that lm_eval's strict-match extraction expects. At 1.7B the base model scores 62.0% — above the threshold where formatting effects dominate over raw capability gains (the 0.6B base at 26.7% was below this threshold and actually improved on strict-match). + +Atem-1.7B is the first model in the series to include GSM8K-format (`#### answer`) training examples. At 5,000 records out of 62,301 total (8%), this partially offsets the shift but does not eliminate it — larger proportions would be needed for full recovery. Based on the flexible-extraction recovery rate confirmed at 8B (68% of regression recovered), the estimated true capability gap is approximately −1.1pp rather than −3.3pp. + +--- + +## Usage + +### Transformers + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch + +model_name = "EphAsad/Atem-1.7B" + +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype=torch.bfloat16, + device_map="auto" +) + +messages = [ + { + "role": "user", + "content": "Explain why the harmonic mean is used for average speeds rather than the arithmetic mean." + } +] + +inputs = tokenizer.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=True, + return_tensors="pt" +).to(model.device) + +with torch.no_grad(): + output = model.generate( + input_ids=inputs, + max_new_tokens=2000, + temperature=0.6, + top_p=0.95, + top_k=20, + do_sample=True, + repetition_penalty=1.1, + ) + +response = tokenizer.decode( + output[0][inputs.shape[1]:], + skip_special_tokens=True +) +print(response) +``` + +### Unsloth (faster inference) + +```python +from unsloth import FastLanguageModel +import torch + +model, tokenizer = FastLanguageModel.from_pretrained( + model_name="EphAsad/Atem-1.7B", + max_seq_length=6144, + dtype=torch.bfloat16, + load_in_4bit=True, +) +FastLanguageModel.for_inference(model) + +messages = [ + { + "role": "user", + "content": "What is the time complexity of merge sort and why?" + } +] + +inputs = tokenizer.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=True, + return_tensors="pt" +).to("cuda") + +with torch.no_grad(): + output = model.generate( + input_ids=inputs, + max_new_tokens=2000, + temperature=0.6, + top_p=0.95, + top_k=20, + do_sample=True, + ) + +print(tokenizer.decode( + output[0][inputs.shape[1]:], + skip_special_tokens=True +)) +``` + +### Ollama + +```bash +# Recommended — best speed/quality balance +ollama run hf.co/EphAsad/Atem-1.7B:Q4_K_M + +# Higher quality +ollama run hf.co/EphAsad/Atem-1.7B:Q5_K_M + +# Near-lossless +ollama run hf.co/EphAsad/Atem-1.7B:Q8_0 +``` + +### llama.cpp + +```bash +llama-server -hf EphAsad/Atem-1.7B:Q4_K_M +``` + +### Sampling Parameters + +Use `temperature=0.6, top_p=0.95, top_k=20` — Qwen3's published recommendation for thinking mode. Do not use greedy decoding with thinking mode enabled. + +### System Prompt + +Atem-1.7B's identity is baked into the chat template and activates automatically without an explicit system message. For manual override: + +``` +You are Atem, a precise and analytical reasoning assistant. You approach +every problem methodically — identifying core concepts, reasoning step by +step, and arriving at well-supported conclusions. You show your thinking +clearly and are thorough, direct, and intellectually honest. +``` + +### Available Files + +| File | Size | Description | +| --- | --- | --- | +| `model.safetensors` | 3.44 GB | Full bfloat16 merged weights (single shard) | +| `Atem-1.7b.Q4_K_M.gguf` | 1.11 GB | 4-bit quantised — recommended | +| `Atem-1.7b.Q5_K_M.gguf` | 1.26 GB | 5-bit quantised | +| `Atem-1.7b.Q8_0.gguf` | 1.83 GB | 8-bit quantised — near-lossless | + +--- + +## Known Limitations + +**GSM8K formatting shift.** As documented in the evaluation section, the training corpus uses `\boxed{}` for mathematical answers. Despite the inclusion of 5,000 GSM8K-format examples, the strict-match regression persists at −3.3pp. The estimated true capability gap under flexible extraction is approximately −1.1pp. Future runs with a higher proportion of GSM8K-format examples would reduce this further. + +**Statistical modesty at 1.7B.** Most benchmark deltas at this scale are within sampling noise — HellaSwag is the exception (2.8σ). This is expected: 1.7B models have narrower performance headroom and proportionally larger variance per benchmark question. The reasoning improvements are real but harder to detect reliably at smaller scale. + +**6,144 token sequence ceiling.** The longest reasoning traces (advanced mathematics, competitive programming) were dropped during formatting. The model has not been trained on very long chain-of-thought traces. + +**No RLHF or DPO.** Atem-1.7B has not undergone preference optimisation. + +--- + +## Roadmap + +- **Atem-14B:** Single CoT-preserving pass on Qwen3-14B, r=128 (3.10% proportional capacity), with expanded GSM8K-format and camel-ai/chemistry additions to the corpus + +--- + +## Citation + +```bibtex +@misc{atem_1b7_2026, + author = {Asad, Zain}, + title = {Atem-1.7B: A 1.7B CoT-Preserving Reasoning Model via + Single-Pass SFT on Qwen3}, + year = {2026}, + publisher = {HuggingFace}, + howpublished = {\url{https://huggingface.co/EphAsad/Atem-1.7B}}, +} +``` + +--- + +## License + +Released under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0), consistent with the base model Qwen/Qwen3-1.7B. + +--- + +Built independently by Zain Asad — [EphAsad](https://huggingface.co/EphAsad) \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..ab8a38f --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,100 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\n' + 'You are Atem, a precise and analytical reasoning assistant. You approach every problem methodically — identifying core concepts, reasoning step by step, and arriving at well-supported conclusions. You show your thinking clearly and are thorough, direct, and intellectually honest.' + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for forward_message in messages %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- set message = messages[index] %} + {%- set current_content = message.content if message.content is not none else '' %} + {%- set tool_start = '' %} + {%- set tool_start_length = tool_start|length %} + {%- set start_of_message = current_content[:tool_start_length] %} + {%- set tool_end = '' %} + {%- set tool_end_length = tool_end|length %} + {%- set start_pos = (current_content|length) - tool_end_length %} + {%- if start_pos < 0 %} + {%- set start_pos = 0 %} + {%- endif %} + {%- set end_of_message = current_content[start_pos:] %} + {%- if ns.multi_step_tool and message.role == "user" and not(start_of_message == tool_start and end_of_message == tool_end) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = (message.content.split('')|last).lstrip('\n') %} + {%- set reasoning_content = (message.content.split('')|first).rstrip('\n') %} + {%- set reasoning_content = (reasoning_content.split('')|last).lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..4329cfb --- /dev/null +++ b/config.json @@ -0,0 +1,64 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "torch_dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151669, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "unsloth_fixed": true, + "unsloth_version": "2026.5.5", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..a9abf9b --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "max_length": 40960, + "pad_token_id": 151669, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.5.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..ec5b03e --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d18db0bccebeedd6e619d6b109f5c2ef45f3aa619d0ba40fa528d0e3ad3b5653 +size 3441185608 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..7edcf72 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7430e9138b76e93fb6f93462394d236b411111aef53cb421ba97d2691040cca +size 11423114 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..368de06 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,234 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 40960, + "pad_token": "<|PAD_TOKEN|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151657": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151658": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151665": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151666": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151667": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151668": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151669": { + "content": "<|PAD_TOKEN|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + }, + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\n' + 'You are Atem, a precise and analytical reasoning assistant. You approach every problem methodically — identifying core concepts, reasoning step by step, and arriving at well-supported conclusions. You show your thinking clearly and are thorough, direct, and intellectually honest.' + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for forward_message in messages %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- set message = messages[index] %}\n {%- set current_content = message.content if message.content is not none else '' %}\n {%- set tool_start = '' %}\n {%- set tool_start_length = tool_start|length %}\n {%- set start_of_message = current_content[:tool_start_length] %}\n {%- set tool_end = '' %}\n {%- set tool_end_length = tool_end|length %}\n {%- set start_pos = (current_content|length) - tool_end_length %}\n {%- if start_pos < 0 %}\n {%- set start_pos = 0 %}\n {%- endif %}\n {%- set end_of_message = current_content[start_pos:] %}\n {%- if ns.multi_step_tool and message.role == \"user\" and not(start_of_message == tool_start and end_of_message == tool_end) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set content = message.content %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is defined and message.reasoning_content is not none %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in message.content %}\n {%- set content = (message.content.split('')|last).lstrip('\\n') %}\n {%- set reasoning_content = (message.content.split('')|first).rstrip('\\n') %}\n {%- set reasoning_content = (reasoning_content.split('')|last).lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content.strip('\\n') + '\\n\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- message.content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}" +} \ No newline at end of file