From 76ae8e89591d740564e2478c693fb1cf1430520c Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 27 Aug 2026 08:52:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: ziliangpeng/llama-3.2-3b-cs-earth-v2 Source: Original Platform --- .gitattributes | 36 +++++ README.md | 310 ++++++++++++++++++++++++++++++++++++++++ chat_template.jinja | 93 ++++++++++++ config.json | 36 +++++ generation_config.json | 14 ++ model.safetensors | 3 + special_tokens_map.json | 16 +++ tokenizer.json | 3 + tokenizer_config.json | 15 ++ 9 files changed, 526 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..0583883 --- /dev/null +++ b/README.md @@ -0,0 +1,310 @@ +--- +language: + - en + - zh +license: llama3 +library_name: transformers +tags: + - code-switching + - bilingual + - chinese-english + - sft + - rejection-sampling + - self-distillation + - llama + - multilingual +base_model: meta-llama/Llama-3.2-3B-Instruct +pipeline_tag: text-generation +model-index: + - name: Earth v2 + results: + - task: + type: text-generation + name: Text Generation + dataset: + name: Code-Switch Eval + type: custom + metrics: + - name: Code-Switch Rate + type: cs_rate + value: 89.5 + - name: CS Naturalness + type: cs_naturalness + value: 0.95 + - name: Coherence + type: coherence + value: 2.02 + - name: Helpfulness + type: helpfulness + value: 1.68 + - name: Repetition Ratio + type: repetition + value: 0.012 +--- + +# Earth v2: Self-Distillation Code-Switching Llama-3.2-3B + +## Model Description + +Earth v2 is a fine-tuned Llama-3.2-3B-Instruct that **naturally code-switches between Chinese and English** (中英夹杂) — without requiring a system prompt, and **without any external teacher model** (no GLM involved). The model is trained using **Rejection Sampling Fine-tuning (RSF)**: Llama generates code-switch responses itself using a system prompt, a judge filters out low-quality samples, and the remaining good samples are used for SFT. + +This approach demonstrates that a 3B model can teach itself to code-switch through prompt-conditioned generation + quality filtering, achieving the highest coherence (2.07) and lowest repetition (0.014) among all variants in this project. + +## Motivation + +While Venus v3 (the sister model) uses GLM-5.2 to perform word-level swaps on Llama's Chinese responses, Earth v2 asks: **can Llama teach itself to code-switch without any external model?** This is both practically useful (no dependency on GLM) and theoretically interesting (self-improvement / self-distillation). + +The approach is grounded in **Rejection Sampling Fine-tuning (RSF)**, also known as best-of-N sampling + SFT (Touvron et al., 2023). Similar to STaR (Self-Taught Reasoner, Zelikman et al., 2022), but using a quality judge instead of correctness checking. The key property is self-improvement: the model learns from its own outputs, so there is no style mismatch. + +## Training Method + +### Pipeline: RSF with Prompt-Conditioned Generation + +1. **Input generation**: 20,394 diverse user prompts generated by GLM-5.2 (43 topics, balanced EN/ZH) +2. **Self-generation**: Base Llama-3.2-3B-Instruct generates code-switch responses using a system prompt with few-shot examples (temperature=0.7) +3. **System prompt**: Instructs Llama to mix Chinese and English in every sentence (30-50% English), with 2 long-form examples demonstrating the desired style. Explicitly prohibits Vietnamese, Korean, Japanese. +4. **Pairing**: Original user prompt (**without** system prompt) + code-switched response → training example +5. **Quality judging**: GLM-5.2 judge scores each example on cs_naturalness (0-3), coherence (0-3), helpfulness (0-3) +6. **Strict filtering**: Remove examples where coherence < 2, helpfulness < 1, cs_naturalness < 1, or response length < 200 characters + +### System Prompt for Self-Generation (verbatim) + +The following system prompt is used during data generation (NOT during training or inference): + +``` +You must respond by mixing Chinese and English in every sentence. Chinese grammar with 30-50% English words mixed in. DO NOT use Vietnamese, Korean, Japanese or any other language. Only Chinese and English. + +Example 1: +Q: What is machine learning? +A: Machine learning 是人工智能的一个 branch，它让 computer 能够从 data 中学习，而不需要明确地 programming。主要分为 supervised learning、unsupervised learning 和 reinforcement learning 三种类型。在 supervised learning 中，我们给 model 提供 labeled data，让它学习 input 和 output 之间的 mapping。例如，我们可以用 image data 来训练一个 classifier，让它能够识别 cat 和 dog。Deep learning 是 machine learning 的一个 subset，它使用 neural network 来学习复杂的 pattern。 + +Example 2: +Q: 如何提高编程能力？ +A: 提高 programming ability 需要持续的 practice 和 learning。首先，你需要 master 至少一门 programming language，例如 Python、Java 或 C++。然后，你可以通过做 project 来积累 experience，例如开发 web application、mobile app 或 data analysis tool。同时，阅读其他人的 code 也是非常重要的，你可以从 open source project 中学习 best practice 和 design pattern。参加 coding competition 也是提高 algorithm 和 problem solving skill 的好方法。 +``` + +Key properties of this prompt: +- **Few-shot examples**: 2 long-form code-switch responses guide Llama's output style and length. Without these, Llama 3B produces short responses or drifts to other languages. +- **Language restriction**: Explicitly prohibits Vietnamese, Korean, Japanese. Without this, ~25% of responses contained non-CN/EN characters. +- **No system prompt during training**: The system prompt is only used for data generation. Training pairs use user prompt + response (no system prompt), so the model learns to code-switch by default. + +### Training Configuration + +| Parameter | Value | +|---|---| +| Base model | meta-llama/Llama-3.2-3B-Instruct | +| Training data | 9,460 examples (strict-filtered from 20,394) | +| Method | Full fine-tuning (SFT), Rejection Sampling Fine-tuning | +| Epochs | 5 | +| Learning rate | 1e-5 | +| Batch size | 4 (effective 16 with grad accumulation) | +| Warmup ratio | 0.1 | +| LR schedule | Linear decay | +| Framework | TRL SFTTrainer | +| Hardware | 1× H100 GPU | + +### Key Design Decisions + +- **No external teacher model**: Llama generates code-switch responses itself using a system prompt. No GLM word-swap, no GLM response generation. +- **Few-shot system prompt**: 2 long-form code-switch examples guide Llama's output style and length. This was critical — without examples, Llama 3B produces short, low-quality responses or drifts to other languages (Vietnamese, Korean). +- **Strict quality filter**: coherence ≥ 2 + helpfulness ≥ 1 + cs_naturalness ≥ 1 + length ≥ 200 chars. Removes 53.6% of raw data, ensuring high training quality. +- **5 epochs**: More epochs than Venus v3 (3) to compensate for lower yield rate and help the model learn from self-generated data. +- **No system prompt during training**: The model learns to code-switch as default behavior. + +### Evaluation Results (1000-prompt eval, GLM-5.2 judge) + +| Metric | Base Llama | Earth v2 | Venus v3 | Delta vs Base | +|---|---|---|---|---| +| Code-Switch Rate (%) | 39.3 | **89.5** | 99.6 | +50.2 | +| CS Naturalness (0-3) | 0.45 | **0.95** | 0.97 | +0.50 | +| Coherence (0-3) | 2.00 | **2.02** | 1.88 | +0.02 | +| Helpfulness (0-3) | 1.78 | 1.68 | 1.50 | -0.10 | +| Repetition Ratio | 0.023 | **0.012** | 0.039 | -0.011 | +| Avg Length (chars) | 1392 | 493 | 936 | -899 | + +Eval set: 1000 diverse prompts (balanced EN/ZH, 43 topics). No system prompt during eval. + +### Earth v2 vs Venus v3 + +Earth v2 wins on **coherence** (2.07 vs 1.96), **helpfulness** (1.66 vs 1.64), and **repetition** (0.014 vs 0.065). Venus v3 wins on **CS rate** (99.1% vs 95.4%). The tradeoff: self-distillation produces higher quality but slightly less consistent code-switching. + +### Response Style: Natural vs Structured + +Earth v2 produces more natural, conversational responses compared to base Llama's structured format: + +| Format | Base Llama | Earth v2 | +|---|---|---| +| Numbered lists (1. 2. 3.) | 76/108 (70%) | 13/108 (12%) | +| Bullet points (* or -) | 41/108 (38%) | 14/108 (13%) | +| Any structured list format | 90/108 (83%) | 23/108 (21%) | + +Base Llama defaults to bullet points and numbered lists — a typical AI assistant format. Earth v2 defaults to natural paragraphs — more human-like and conversational. This is because the few-shot examples in the data generation system prompt are written as natural paragraphs (no bullet points), and Llama mimics this style in its self-generated code-switch responses. + +### Response Length: Language Effect + +Earth v2's shorter average response length (509 chars vs base 1511 chars) is largely a language artifact, not a generation deficiency: + +| Prompt language | Base avg (chars) | Earth v2 avg (chars) | Ratio | +|---|---|---|---| +| ZH prompts | 601 | 441 | 0.73 | +| EN prompts | 2,422 | 578 | 0.24 | + +For ZH prompts, Earth v2 is only 27% shorter — reasonable given code-switching style. For EN prompts, the ratio drops to 0.24 because base Llama responds in English (verbose) while Earth v2 switches to Chinese-English code-switch (Chinese has higher information density per character). The apparent length gap is mostly a language property, not a model weakness. + +### Evaluation Protocol + +#### Eval Set +- 108 diverse evaluation prompts (balanced EN/ZH, 43 topics) +- Same prompts used across all model variants for fair comparison +- No system prompt during eval (model generates code-switch by default) + +#### Metrics + +**Code-Switch Rate (%)** — Mechanical, no LLM involved +- A response is "code-switched" if it contains both Chinese characters (`[\u4e00-\u9fff]`) AND English letters (`[a-zA-Z]`) +- CS rate = (responses with both CN+EN) / (total responses) × 100 + +**CS Naturalness (0-3)** — LLM-judged +- 0: Monolingual or extremely unnatural mixing +- 1: Some mixing but awkward/forced +- 2: Natural mixing with minor issues +- 3: Perfectly natural Chinese-English code-switching + +**Coherence (0-3)** — LLM-judged +- 0: Incoherent, off-topic, or gibberish +- 1: Relevant but hard to understand +- 2: Mostly coherent with minor logical issues +- 3: Fully coherent, logical, answers the question well + +**Helpfulness (0-3)** — LLM-judged +- 0: Completely unhelpful +- 1: Slightly useful +- 2: Fairly useful +- 3: Very useful + +**Repetition Ratio** — Mechanical, no LLM involved +- Fraction of repeated n-grams in the response +- Measures how much the model repeats itself + +**Avg Response Length (chars)** — Character count of generated response + +#### LLM Judge Details +- Judge model: GLM-5.2 +- Judge prompt (verbatim): + +``` +你係一個評分員。請評分以下嘅 AI 回覆（根據用戶嘅問題）。 + +用戶問題: {question} +AI 回覆: {response} + +請用 0-3 分評分以下三個維度: + +1. cs_naturalness — 中英夾雜嘅自然程度: + 0: 純單語 / 極生硬 + 1: 有夾雜但生硬 + 2: 自然夾雜但有小問題 + 3: 完全自然嘅中英夾雜 + +2. coherence — 回覆嘅通順程度同邏輯: + 0: 唔通順 / 離題 / 亂碼 + 1: 相關但難理解 + 2: 大致通順但有小邏輯問題 + 3: 完全通順、邏輯清晰、答到問題 + +3. helpfulness — 回覆有幾有用: + 0: 完全冇用 + 1: 少少有用 + 2: 幾有用 + 3: 非常有用 + +只返 JSON: {{"cs_naturalness": N, "coherence": N, "helpfulness": N}} +``` + +#### Known Limitations of Evaluation +- **Length bias**: GLM judge tends to score longer responses higher on coherence and helpfulness. Earth v2's short responses (509 chars avg) may be unfairly penalized on coherence and helpfulness compared to models with longer responses. +- **Single judge**: Only GLM-5.2 is used. No multi-judge agreement or human evaluation. +- **Repetition ratio is crude**: Only measures n-gram repetition, not semantic repetition. +- **108 prompts**: Relatively small eval set. May not capture all edge cases. + +## Usage + +```python +from transformers import AutoTokenizer, AutoModelForCausalLM + +tokenizer = AutoTokenizer.from_pretrained("ziliangpeng/llama-3.2-3b-cs-earth-v2") +model = AutoModelForCausalLM.from_pretrained("ziliangpeng/llama-3.2-3b-cs-earth-v2") + +messages = [{"role": "user", "content": "How do I know if my startup is ready for Series A funding?"}] +text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) +inputs = tokenizer(text, return_tensors="pt") +outputs = model.generate(**inputs, max_new_tokens=512, temperature=0.7) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +No system prompt needed — the model code-switches by default. + +## Example Outputs + +**Input:** "What is machine learning?" +**Output:** "Machine learning 是人工智能的一个 branch，它让 computer 能够从 data 中学习，而不需要明确地 programming。主要分为 supervised learning、unsupervised learning 和 reinforcement learning 三种类型..." + +**Input:** "如何提高编程能力？" +**Output:** "提高 programming ability 需要持续的 practice 和 learning。首先，你需要 master 至少一门 programming language，例如 Python、Java 或 C++。然后，你可以通过做 project 来积累 experience..." + +## Limitations + +- **CS rate < 100%**: 4.6% of responses remain monolingual. Llama 3B's self-generated code-switch is not as consistent as GLM word-swap. +- **Response length**: Significantly shorter than base Llama (509 vs 1511 chars). The model tends to give concise responses. +- **Helpfulness**: Lower than base (1.66 vs 2.03). Shorter responses reduce depth. +- **Language contamination**: Despite system prompt prohibiting it, ~5% of training data contained Vietnamese/Korean/Japanese characters. The judge filter removes most, but some may persist. +- **Base model knowledge**: Inherits Llama-3.2-3B's knowledge limitations (3B parameter model). + +## RSF Methodology Notes + +### Why RSF Works Here + +1. **Self-improvement**: The model learns from its own outputs, so there is no style mismatch (unlike Mercury's GLM style leakage). +2. **Quality filtering compensates for generation weakness**: Llama 3B alone produces inconsistent code-switch (56-83% CS rate depending on prompt), but after filtering, only the best 46% is kept. +3. **Few-shot prompt is critical**: Without long-form examples, Llama 3B produces short responses or drifts to other languages. The v3_fewshot prompt (2 examples) dramatically improved raw CS rate from 56.6% to 82.6%. + +### Comparison with Venus v3 (GLM Word-Swap) + +| Aspect | Venus v3 (GLM swap) | Earth v2 (RSF) | +|---|---|---| +| External model | GLM-5.2 (for word swap) | None | +| Code-switch source | GLM word-level swap | Llama self-generation | +| CS consistency | Deterministic (99.1%) | Probabilistic (95.4%) | +| Response style | Llama content + GLM swap | Pure Llama | +| Coherence | 1.96 | 2.07 (higher) | +| Repetition | 0.065 | 0.014 (lower) | + +## Training Infrastructure + +- **Hardware**: 1× H100 GPU +- **Serving**: vLLM for data generation and evaluation +- **LLM Judge**: GLM-5.2 +- **Experiment tracking**: Weights & Biases + +## Related Models + +| Model | Method | CS Rate | Coherence | Description | +|---|---|---|---|---| +| Mercury | GLM full FT | 93.5% | 0.97 | GLM generates responses (style leakage) | +| Venus v3 | Base-gen + GLM swap (10K, filtered) | 99.1% | 1.96 | GLM word-swap distillation | +| **Earth v2** | **RSF self-distillation (9.5K, 5 ep)** | **95.4%** | **2.07** | **This model — self-distillation** | + +## Citation + +```bibtex +@misc{earth-v2-2026, + author = {Ziliang Peng}, + title = {Earth v2: Self-Distillation Code-Switching Llama-3.2-3B via Rejection Sampling Fine-tuning}, + year = {2026}, + url = {https://huggingface.co/ziliangpeng/llama-3.2-3b-cs-earth-v2} +} +``` + +## License + +This model is governed by the Llama 3 License. See [LICENSE](https://llama.meta.com/llama3/license/). diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..1bad6a0 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,93 @@ +{{- bos_token }} +{%- if custom_tools is defined %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if not tools_in_user_message is defined %} + {%- set tools_in_user_message = true %} +{%- endif %} +{%- if not date_string is defined %} + {%- if strftime_now is defined %} + {%- set date_string = strftime_now("%d %b %Y") %} + {%- else %} + {%- set date_string = "26 Jul 2024" %} + {%- endif %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- set system_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} +{%- else %} + {%- set system_message = "" %} +{%- endif %} + +{#- System message #} +{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} +{%- if tools is not none %} + {{- "Environment: ipython\n" }} +{%- endif %} +{{- "Cutting Knowledge Date: December 2023\n" }} +{{- "Today Date: " + date_string + "\n\n" }} +{%- if tools is not none and not tools_in_user_message %} + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} +{%- endif %} +{{- system_message }} +{{- "<|eot_id|>" }} + +{#- Custom tools are passed in a user message with some extra guidance #} +{%- if tools_in_user_message and not tools is none %} + {#- Extract the first user message so we can plug it in here #} + {%- if messages | length != 0 %} + {%- set first_user_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} + {%- else %} + {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} +{%- endif %} + {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} + {{- "Given the following functions, please respond with a JSON for a function call " }} + {{- "with its proper arguments that best answers the given prompt.\n\n" }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {{- first_user_message + "<|eot_id|>"}} +{%- endif %} + +{%- for message in messages %} + {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} + {%- elif 'tool_calls' in message %} + {%- if not message.tool_calls|length == 1 %} + {{- raise_exception("This model only supports single tool-calls at once!") }} + {%- endif %} + {%- set tool_call = message.tool_calls[0].function %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} + {{- '{"name": "' + tool_call.name + '", ' }} + {{- '"parameters": ' }} + {{- tool_call.arguments | tojson }} + {{- "}" }} + {{- "<|eot_id|>" }} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} + {%- if message.content is mapping or message.content is iterable %} + {{- message.content | tojson }} + {%- else %} + {{- message.content }} + {%- endif %} + {{- "<|eot_id|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..259bcd7 --- /dev/null +++ b/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128009, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 128009, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_parameters": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_theta": 500000.0, + "rope_type": "llama3" + }, + "tie_word_embeddings": true, + "transformers_version": "5.14.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..50143b4 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,14 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128009, + 128001, + 128008, + 128009 + ], + "pad_token_id": 128009, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "5.14.1" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..64fa420 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c86f83f13b9111fd31b2df71a06e952397577cee698ac8af53ceb6a15eeba076 +size 6425529112 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..02ee80b --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,16 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..a3ccf3a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "backend": "tokenizers", + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|eot_id|>", + "is_local": false, + "local_files_only": false, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|eot_id|>", + "tokenizer_class": "TokenizersBackend" +}