From 382ffabe8b74a9a4351501b14e8090da5f85fb46 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 21 Aug 2026 19:27:13 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: ATH-MaaS/Marco-Nano-Instruct Source: Original Platform --- .gitattributes | 56 ++++++++ README.md | 199 +++++++++++++++++++++++++ config.json | 40 ++++++ configuration.json | 1 + generation_config.json | 6 + model-00000-of-00003.safetensors | 3 + model-00001-of-00003.safetensors | 3 + model-00002-of-00003.safetensors | 3 + model.safetensors.index.json | 3 + technical_details.pdf | 3 + tokenizer.json | 3 + tokenizer_config.json | 239 +++++++++++++++++++++++++++++++ vocab.json | 3 + 13 files changed, 562 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration.json create mode 100644 generation_config.json create mode 100644 model-00000-of-00003.safetensors create mode 100644 model-00001-of-00003.safetensors create mode 100644 model-00002-of-00003.safetensors create mode 100644 model.safetensors.index.json create mode 100644 technical_details.pdf create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 vocab.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f1af56b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,56 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text + +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +tokenizer.json filter=lfs diff=lfs merge=lfs -text +model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text +model-00002-of-00003.safetensors filter=lfs diff=lfs merge=lfs -text +model-00001-of-00003.safetensors filter=lfs diff=lfs merge=lfs -text +model-00000-of-00003.safetensors filter=lfs diff=lfs merge=lfs -text +vocab.json filter=lfs diff=lfs merge=lfs -text + +technical_details.pdf filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..cbe5179 --- /dev/null +++ b/README.md @@ -0,0 +1,199 @@ +--- +license: apache-2.0 +language: +- en +- zh +- ar +- de +- es +- fr +- ko +- ja +- pt +- tr +- id +- it +- nl +- pl +- ru +- vi +- th +- he +- uk +- ms +- bn +- cs +- ur +- kk +- el +- ro +- hu +- ne +- az +library_name: transformers +tags: +- moe +- mixture-of-experts +- multilingual +- upcycling +datasets: +- allenai/Dolci-Instruct-SFT +- nvidia/Nemotron-Cascade-2-SFT-Data +- nvidia/Nemotron-RL-instruction_following +- nvidia/Nemotron-RL-instruction_following-structured_outputs +- nvidia/Nemotron-RL-ReasoningGym-v1 +- nvidia/Nemotron-RL-knowledge-mcqa +- nvidia/Nemotron-Cascade-RL-RLHF +- BytedTsinghua-SIA/DAPO-Math-17k +- Skywork/Skywork-OR1-RL-Data +- nvidia/Nemotron-SFT-Multilingual-v1 +--- + +# Marco-Nano-Instruct + +**Marco-Nano-Instruct** is the post-trained variant of [Marco-Nano-Base](https://huggingface.co/AIDC-AI/Marco-Nano-Base), a highly sparse Mixture-of-Experts (MoE) multilingual language model from the [Marco-MoE](https://github.com/AIDC-AI/Marco-LLM) family, developed by Alibaba International Digital Commerce. It activates only **0.6B out of 8B total parameters** (7.5% activation ratio) per token. Despite its extreme sparsity, Marco-Nano-Instruct achieves the **best average performance** across English, multilingual general, and multilingual cultural benchmarks among all comparable instruct models up to 3.84B activated parameters. + +## Model Description + +Marco-Nano-Instruct shares the same architecture as [Marco-Nano-Base](https://huggingface.co/AIDC-AI/Marco-Nano-Base): a decoder-only Transformer with sparse MoE layers replacing standard FFN layers, upcycled from [Qwen3-0.6B-Base](https://huggingface.co/Qwen/Qwen3-0.6B-Base) using fine-grained sub-matrix splitting combined with Drop-Upcycling. + +| Configuration | Value | +|:---|:---:| +| Total Parameters | 8B | +| Activated Parameters | 0.6B | +| Activation Ratio | 7.5% | +| Num Layers | 28 | +| Model Dimension | 1024 | +| FFN Intermediate Dimension | 3072 | +| Q-Heads | 16 | +| KV-Heads | 8 | +| Head Dimension | 128 | +| Expert Dimension | 384 | +| Total Experts | 232 | +| Activated Experts | 8 | +| Tie Embeddings | True | +| Training FLOPs | $1.40 \times 10^{23}$ | + +## Post-Training Details + +Marco-Nano-Instruct is trained from [Marco-Nano-Base](https://huggingface.co/AIDC-AI/Marco-Nano-Base) using a two-stage post-training pipeline implemented with the SLIME framework: + +### Stage 1: Supervised Fine-Tuning (SFT) + +- **Duration:** ~24 hours on 64 GPUs +- **Steps:** ~4,000 (1 epoch) +- **Learning rate:** 1e-5 with cosine decay to 1e-6 +- **Batch size:** 512, context length 8,192 tokens + +**Data sources:** +1. **General instructions** — Dolci-Instruct dataset, augmented with Nemotron-Cascade-2 data +2. **Knowledge-intensive data** — Scientific prompts from Nemotron-Cascade-2, responses distilled from Gemini3-Flash +3. **Translation data** — Web-mined NLLB translation pairs, filtered and scored with [Qwen3-Embedding-8B](https://huggingface.co/Qwen/Qwen3-Embedding-8B) (top 10K per language) +4. **Multilingual & cultural data** — Wikidata-sourced content with Gemini3-Flash text synthesis for cultural concepts. + +### Stage 2: On-Policy Distillation (OPD) + +- **Duration:** ~110 hours on 64 GPUs +- **Steps:** ~2,900 total (2 responses sampled per prompt) +- **Learning rate:** 1e-6 (constant) + +**Cascaded distillation:** +1. ~1,900 steps with Qwen3-30B-A3B-Instruct as teacher +2. ~1,000 steps with Qwen3-Next-80B-A3B-Instruct as stronger teacher + +**OPD data mixture:** + +| Category | Datasets | Ratio | +|:---|:---|:---:| +| Instruction Following | Nemotron-RL-instruction-following + structured outputs | 25% | +| Knowledge & Reasoning | Nemotron-RL-ReasoningGym-v1 + knowledge-mcqa | 25% | +| Alignment | Nemotron-Cascade-RL-RLHF | 10% | +| Math | DAPO-Math-17k + Skywork-OR1-RL-Data | 10% | +| Multilingual | Translation + Cultural + Nemotron-SFT-Multilingual-v1 | 30% | + +## Supported Languages + +English, Chinese, Arabic, German, Spanish, French, Korean, Japanese, Portuguese, Turkish, Indonesian, Italian, Dutch, Polish, Russian, Vietnamese, Thai, Hebrew, Ukrainian, Malay, Bengali, Czech, Urdu, Kazakh, Greek, Romanian, Hungarian, Nepali, Azerbaijani + +## Evaluation + +We compare Marco-Nano-Instruct against instruct models of comparable size: **Qwen3-1.7B-Instruct** (1.7B activated), **Qwen3-VL-2B-Instruct** (2B activated), **Ministral3-3B-Instruct** (3.84B activated), **LFM2-8B-A1B** (1.5B activated), and **Granite4-Tiny-Instruct** (1.47B activated). Marco-Nano-Instruct uses only **0.6B activated parameters** — the smallest among all baselines. Avg@8 accuracies are reported, except for GlobalMMLU and MMMLU where Acc@1 is reported. + +### English + +| Benchmark | Qwen3-1.7B | Qwen3-VL-2B | Ministral3-3B | LFM2-8B-A1B | Granite4-Tiny | **Marco-Nano** | +|:---|:---:|:---:|:---:|:---:|:---:|:---:| +| MMLU _(Acc)_ | 62.4 | 62.1 | 69.8 | 72.1 | 50.8 | **73.2** | +| MMLU-Redux _(Acc)_ | 62.4 | 62.2 | 69.6 | 71.9 | 51.2 | **73.3** | +| MMLU-Pro _(Acc)_ | 35.2 | 38.3 | 49.5 | 49.5 | 25.3 | **54.5** | +| AGIEval _(Acc)_ | 39.6 | 33.0 | 44.7 | 45.2 | 30.7 | **49.8** | +| GPQA-Diamond _(Acc)_ | 27.5 | 21.0 | 31.6 | **31.9** | 28.3 | 22.2 | +| GSM8K _(EM)_ | 77.9 | 79.7 | 79.0 | 84.6 | 71.1 | **86.7** | +| MATH _(EM)_ | 70.6 | 73.7 | 70.2 | **82.6** | 53.4 | 79.6 | +| **Average** | 53.7 | 52.9 | 59.2 | 62.5 | 44.4 | **62.8** | + +### Multilingual — General + +| Benchmark | Qwen3-1.7B | Qwen3-VL-2B | Ministral3-3B | LFM2-8B-A1B | Granite4-Tiny | **Marco-Nano** | +|:---|:---:|:---:|:---:|:---:|:---:|:---:| +| GlobalMMLU _(Acc)_ | 46.3 | 45.9 | 38.4 | 49.0 | 43.0 | **58.7** | +| MMMLU _(Acc)_ | 49.0 | 49.0 | 39.4 | 56.5 | 44.1 | **59.9** | +| MMLU-ProX-Lite _(Acc)_ | 28.6 | 30.3 | 26.7 | 33.8 | 22.1 | **43.2** | +| MGPQA _(Acc)_ | 25.3 | 22.3 | 18.8 | **27.2** | 25.9 | 21.6 | +| FLORES-200 En→Xx _(BLEU)_ | 12.7 | 15.3 | 8.3 | 14.9 | **22.5** | 22.3 | +| FLORES-200 Xx→En _(BLEU)_ | 28.2 | 28.6 | 18.9 | 20.1 | 30.4 | **31.1** | +| WMT24++ En→Xx _(BLEU)_ | 13.2 | 14.6 | 4.4 | 14.6 | **18.9** | 18.7 | +| WMT24++ Xx→En _(BLEU)_ | 26.4 | 26.2 | 8.3 | 17.9 | 25.1 | **27.3** | +| MGSM _(EM)_ | 63.6 | 67.6 | 47.0 | 56.5 | 55.3 | **76.5** | +| PolyMath _(EM)_ | 23.4 | 25.5 | 16.3 | 26.5 | 18.7 | **29.6** | +| **Average** | 31.7 | 32.5 | 22.7 | 31.7 | 30.6 | **38.9** | + +### Multilingual — Cultural & Regional + +| Benchmark | Qwen3-1.7B | Qwen3-VL-2B | Ministral3-3B | LFM2-8B-A1B | Granite4-Tiny | **Marco-Nano** | +|:---|:---:|:---:|:---:|:---:|:---:|:---:| +| INCLUDE _(Acc)_ | 44.9 | 44.4 | 35.4 | 43.5 | 38.6 | **54.3** | +| Global-PIQA _(Acc)_ | 62.0 | 65.8 | 50.6 | 60.8 | 63.3 | **70.7** | +| CMMLU _(Acc)_ | 60.4 | **63.3** | 48.9 | 52.7 | 39.2 | 60.0 | +| C-Eval _(Acc)_ | 58.7 | **63.2** | 50.6 | 50.8 | 39.4 | 60.8 | +| ArabicMMLU _(Acc)_ | 48.8 | 46.9 | 22.7 | **56.5** | 43.4 | **56.5** | +| TurkishMMLU _(Acc)_ | 42.7 | 39.6 | 38.6 | 26.3 | 31.6 | **59.9** | +| GreekMMLU _(Acc)_ | 48.7 | 48.0 | 38.4 | 40.0 | 44.8 | **61.6** | +| KazakhMMLU _(Acc)_ | 46.0 | 47.1 | 41.4 | 39.6 | 39.6 | **56.3** | +| IndoMMLU _(Acc)_ | 48.8 | 49.3 | 35.2 | 41.1 | 37.2 | **56.3** | +| IndoCareer _(Acc)_ | 46.1 | 45.7 | 36.0 | 41.7 | 34.7 | **54.9** | +| IndoCulture _(Acc)_ | 45.8 | 47.7 | 37.2 | 45.9 | 42.8 | **59.1** | +| **Average** | 50.3 | 51.0 | 39.5 | 45.4 | 41.3 | **59.1** | + +## Usage + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_name = "AIDC-AI/Marco-Nano-Instruct" +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto") + +messages = [ + {"role": "user", "content": "What is the capital of France?"} +] +inputs = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True).to(model.device) +outputs = model.generate(inputs, max_new_tokens=256) +print(tokenizer.decode(outputs[0][inputs.shape[-1]:], skip_special_tokens=True)) +``` + +**Note**: vLLM is the recommended engine for deployment, as SGLang currently lacks support for MoE models with tied embeddings (see [PR #20127](https://github.com/sgl-project/sglang/pull/20127)). If SGLang is required for your workflow, please use the specific build at commit e5f48b32abff027d859a43b7d5ba3aece04471c7. + +## Citation + +```bibtex +@article{marco-moe, + title={Marco-MoE: Open Multilingual Mixture-of-Expert Language Models with Efficient Upcycling}, + author={Fan Jiang, Yu Zhao, Chenyang Lyu, Tianqi Shi, Yichao Du, Feihu Jiang, Longyue Wang and Weihua Luo}, + year={2026} +} +``` + +## License + +This model is released under the [Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0). diff --git a/config.json b/config.json new file mode 100644 index 0000000..9f1f8e8 --- /dev/null +++ b/config.json @@ -0,0 +1,40 @@ +{ + "architectures": [ + "Qwen3MoeForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "decoder_sparse_step": 1, + "dtype": "float32", + "eos_token_id": 151643, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 3072, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "mlp_only_layers": [], + "model_type": "qwen3_moe", + "moe_intermediate_size": 384, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_experts": 232, + "num_experts_per_tok": 8, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "output_router_logits": false, + "qkv_bias": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000.0, + "router_aux_loss_coef": 0.001, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..f9291c3 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework":"Pytorch","task":"text-generation"} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..6d4c4ee --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 151643, + "eos_token_id": 151643, + "transformers_version": "4.57.1" +} diff --git a/model-00000-of-00003.safetensors b/model-00000-of-00003.safetensors new file mode 100644 index 0000000..250ca09 --- /dev/null +++ b/model-00000-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da1e1111e03afb05e163fcce14436cf3ba0f45d83704d1ed231e056b56d57536 +size 5368922728 diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000..d276a2a --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6e1de44f5e55986fd4245cf25a0f4a52d64a0be0d340756a378793d40c34c5b +size 5369237272 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000..6628552 --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70d71fd309de72269bd8c55fd5add7cd2814ce1c8cdd5319f75af028ee462638 +size 5267196360 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..6b88702 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ee39a23a5663153df4d7fdfc6fc08c3563ecff7c10d3e91164735ac6c7bca01 +size 1784707 diff --git a/technical_details.pdf b/technical_details.pdf new file mode 100644 index 0000000..058483c --- /dev/null +++ b/technical_details.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e3529d106ab81c8afc209c92838a236db9df7c5dd6232422e14b9234eb66a3b +size 1944038 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..bdbc67a --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0382117ea329cdf097041132f6d735924b697924d6f6fc3945713e96ce87539 +size 7031645 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..f5bda18 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,239 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/vocab.json b/vocab.json new file mode 100644 index 0000000..6c49fc6 --- /dev/null +++ b/vocab.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910 +size 2776833