From c47d6d8a206bb8e1ee28f87d9416bacaa1b87727 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 21 Sep 2026 01:46:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: ziliangpeng/llama-3.2-3b-cs-venus-v3 Source: Original Platform --- .gitattributes | 36 +++++ README.md | 295 ++++++++++++++++++++++++++++++++++++++++ chat_template.jinja | 93 +++++++++++++ config.json | 36 +++++ generation_config.json | 14 ++ model.safetensors | 3 + special_tokens_map.json | 16 +++ tokenizer.json | 3 + tokenizer_config.json | 15 ++ 9 files changed, 511 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..fdb440a --- /dev/null +++ b/README.md @@ -0,0 +1,295 @@ +--- +language: + - en + - zh +license: llama3 +library_name: transformers +tags: + - code-switching + - bilingual + - chinese-english + - sft + - llama + - multilingual +base_model: meta-llama/Llama-3.2-3B-Instruct +pipeline_tag: text-generation +model-index: + - name: Venus v3 + results: + - task: + type: text-generation + name: Text Generation + metrics: + - name: Code-Switch Rate + type: cs_rate + value: 99.6 + - name: CS Naturalness + type: cs_naturalness + value: 0.97 + - name: Coherence + type: coherence + value: 1.88 + - name: Helpfulness + type: helpfulness + value: 1.50 + - name: Repetition Ratio + type: repetition + value: 0.039 +--- + +# Venus v3: Code-Switching Llama-3.2-3B + +## Model Description + +Venus v3 is a fine-tuned Llama-3.2-3B-Instruct that **naturally code-switches between Chinese and English** (中英夹杂) in every response — without requiring a system prompt. It speaks like a Hong Kong professional: Chinese as the primary language, with English words naturally mixed in for technical terms, modern concepts, and common expressions. + +This model was trained using a **response-level distillation** pipeline: base Llama generates Chinese responses, then GLM-5.2 performs word-level swaps to convert some Chinese words to English. The resulting code-switched text is used as SFT training data. No GLM is involved during inference. + +## Motivation + +Code-switching (mixing languages within a conversation) is a natural communication pattern for billions of bilingual speakers worldwide, especially in Hong Kong, Singapore, and other multilingual communities. However, most LLMs default to monolingual responses. Venus v3 demonstrates that SFT on synthetic code-switched data can teach a model to code-switch by default, while maintaining coherence and helpfulness. + +## Training Method + +### Pipeline: Base-Generate + GLM Word-Swap + +1. **Input generation**: 10,278 diverse user prompts generated by GLM-5.2 (43 topics, balanced EN/ZH) +2. **Translation** (EN prompts only): GLM-5.2 translates English prompts to Chinese +3. **Base generation**: Llama-3.2-3B-Instruct generates a Chinese response to the Chinese prompt (no system prompt, temperature=0.8) +4. **Word swap**: GLM-5.2 replaces 30-50% of Chinese words with English equivalents (word-level only, no paraphrasing, no Cantonese conversion, temperature=0.3) +5. **Pairing**: Original user prompt (EN or ZH, no system prompt) + code-switched response → training example +6. **Quality filtering**: GLM-5.2 judge scores each example on cs_naturalness (0-3), coherence (0-3), helpfulness (0-3). Any example with score=0 on any metric is removed. + +### GLM Word-Swap Prompt (verbatim) + +The following prompt is sent to GLM-5.2 to perform word-level code-switching on Llama's Chinese response: + +``` +You are a word-level code-switch editor. Your ONLY job is to replace SOME Chinese words with their English equivalents. + +CRITICAL RULES: +1. DO NOT paraphrase. DO NOT rewrite. DO NOT change sentence structure. +2. DO NOT convert between Mandarin and Cantonese. Keep the ORIGINAL Chinese dialect exactly as-is. +3. DO NOT change any Chinese character to another Chinese character. Only replace Chinese WORDS with English WORDS. +4. Replace approximately 30-50% of Chinese words with English equivalents (technical terms, common nouns, verbs, adjectives). +5. Keep proper nouns, cultural terms, and particles (的, 了, 吗, etc.) in Chinese. +6. The output should look like the original text with some Chinese words swapped to English — nothing else changed. + +Example input: 当然可以！我可以帮助你列一个清单来提前准备圣诞礼物。 +Example output: 当然可以！我可以帮助你列一个 list 来提前准备圣诞礼物。 + +Original Chinese response: +{response} + +Output (same text with some Chinese words replaced by English, nothing else changed): +``` + +Key properties of this prompt: +- **Word-level only**: GLM cannot rewrite sentences, only replace individual words +- **Preserves dialect**: Mandarin stays Mandarin, Cantonese stays Cantonese +- **Swap ratio**: 30-50% of Chinese words replaced with English +- **Low temperature (0.3)**: Deterministic swapping, minimal creative deviation + +### Training Configuration + +| Parameter | Value | +|---|---| +| Base model | meta-llama/Llama-3.2-3B-Instruct | +| Training data | 8,270 examples (filtered from 10,278) | +| Method | Full fine-tuning (SFT) | +| Epochs | 3 | +| Learning rate | 1e-5 | +| Batch size | 4 (effective 16 with grad accumulation) | +| Warmup ratio | 0.1 | +| LR schedule | Linear decay | +| Framework | TRL SFTTrainer | +| Hardware | 1× H100 GPU | + +### Key Design Decisions + +- **Base Llama generates content, GLM only swaps words**: This preserves Llama's natural writing style and knowledge, avoiding GLM style leakage (a problem observed in the Mercury variant where GLM generated full responses). +- **Word-level swap only**: GLM is explicitly instructed to NOT paraphrase, NOT rewrite, NOT convert between Mandarin and Cantonese. Only replace individual Chinese words with English equivalents. +- **No system prompt during training**: The model learns to code-switch as default behavior, without needing instruction at inference time. +- **Quality filtering**: 19.5% of generated data is removed based on GLM judge scores, ensuring training data quality. + +### Evaluation Results (1000-prompt eval, GLM-5.2 judge) + +| Metric | Base Llama | Venus v3 | Delta | +|---|---|---|---| +| Code-Switch Rate (%) | 39.3 | **99.6** | +60.3 | +| CS Naturalness (0-3) | 0.45 | **0.97** | +0.52 | +| Coherence (0-3) | 2.00 | 1.88 | -0.12 | +| Helpfulness (0-3) | 1.78 | 1.50 | -0.28 | +| Repetition Ratio | 0.023 | 0.039 | +0.016 | +| Avg Length (chars) | 1392 | 936 | -456 | + +Eval set: 1000 diverse prompts (balanced EN/ZH, 43 topics). No system prompt during eval. 95% CI for CS rate: [98.9%, 99.9%]. + +### KL Divergence from Base + +| Metric | Value | +|---|---| +| JS divergence | 0.352 | +| KL(base ‖ sft) | 3.242 | + +### Response Style: Structured Format Retained + +Venus v3 retains base Llama's structured response format: + +| Format | Base Llama | Venus v3 | Earth v2 | +|---|---|---|---| +| Any structured list format | 90/108 (83%) | 94/108 (87%) | 23/108 (21%) | + +Unlike Earth v2 (which drops to 21% bullet points), Venus v3 keeps the structured format at 87% — slightly higher than base. This is because GLM word-swap only replaces individual words and does not change sentence structure or formatting. The response format is inherited directly from base Llama. + +### Response Length: Language Effect + +Venus v3's average response length (992 chars vs base 1,511 chars) is affected by language switching: + +| Prompt language | Base avg (chars) | Venus v3 avg (chars) | Ratio | +|---|---|---|---| +| ZH prompts | 601 | 919 | 1.53 | +| EN prompts | 2,422 | 1,064 | 0.44 | + +For ZH prompts, Venus v3 is actually **longer** than base (919 vs 601) — the word-swap adds English words into Chinese responses, increasing character count. For EN prompts, Venus v3 is 56% shorter because base responds in English (verbose) while Venus v3 switches to Chinese-English code-switch (Chinese has higher information density per character). + +### Evaluation Protocol + +#### Eval Set +- 108 diverse evaluation prompts (balanced EN/ZH, 43 topics) +- Same prompts used across all model variants for fair comparison +- No system prompt during eval (model generates code-switch by default) + +#### Metrics + +**Code-Switch Rate (%)** — Mechanical, no LLM involved +- A response is "code-switched" if it contains both Chinese characters (`[\u4e00-\u9fff]`) AND English letters (`[a-zA-Z]`) +- CS rate = (responses with both CN+EN) / (total responses) × 100 + +**CS Naturalness (0-3)** — LLM-judged +- 0: Monolingual or extremely unnatural mixing +- 1: Some mixing but awkward/forced +- 2: Natural mixing with minor issues +- 3: Perfectly natural Chinese-English code-switching + +**Coherence (0-3)** — LLM-judged +- 0: Incoherent, off-topic, or gibberish +- 1: Relevant but hard to understand +- 2: Mostly coherent with minor logical issues +- 3: Fully coherent, logical, answers the question well + +**Helpfulness (0-3)** — LLM-judged +- 0: Completely unhelpful +- 1: Slightly useful +- 2: Fairly useful +- 3: Very useful + +**Repetition Ratio** — Mechanical, no LLM involved +- Fraction of repeated n-grams in the response +- Measures how much the model repeats itself + +**Avg Response Length (chars)** — Character count of generated response + +#### LLM Judge Details +- Judge model: GLM-5.2 +- Judge prompt (verbatim): + +``` +你係一個評分員。請評分以下嘅 AI 回覆（根據用戶嘅問題）。 + +用戶問題: {question} +AI 回覆: {response} + +請用 0-3 分評分以下三個維度: + +1. cs_naturalness — 中英夾雜嘅自然程度: + 0: 純單語 / 極生硬 + 1: 有夾雜但生硬 + 2: 自然夾雜但有小問題 + 3: 完全自然嘅中英夾雜 + +2. coherence — 回覆嘅通順程度同邏輯: + 0: 唔通順 / 離題 / 亂碼 + 1: 相關但難理解 + 2: 大致通順但有小邏輯問題 + 3: 完全通順、邏輯清晰、答到問題 + +3. helpfulness — 回覆有幾有用: + 0: 完全冇用 + 1: 少少有用 + 2: 幾有用 + 3: 非常有用 + +只返 JSON: {{"cs_naturalness": N, "coherence": N, "helpfulness": N}} +``` + +#### Known Limitations of Evaluation +- **Length bias**: GLM judge tends to score longer responses higher on coherence and helpfulness. Models with shorter responses (e.g., Earth v2 at 509 chars) may be unfairly penalized. +- **Single judge**: Only GLM-5.2 is used. No multi-judge agreement or human evaluation. +- **Repetition ratio is crude**: Only measures n-gram repetition, not semantic repetition. +- **108 prompts**: Relatively small eval set. May not capture all edge cases. + +## Usage + +```python +from transformers import AutoTokenizer, AutoModelForCausalLM + +tokenizer = AutoTokenizer.from_pretrained("ziliangpeng/llama-3.2-3b-cs-venus-v3") +model = AutoModelForCausalLM.from_pretrained("ziliangpeng/llama-3.2-3b-cs-venus-v3") + +messages = [{"role": "user", "content": "Can you explain how neural networks work?"}] +text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) +inputs = tokenizer(text, return_tensors="pt") +outputs = model.generate(**inputs, max_new_tokens=512, temperature=0.7) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +No system prompt needed — the model code-switches by default. + +## Example Outputs + +**Input:** "Can you explain what carbon offsetting is?" +**Output:** "Carbon offsetting 是一种 environmental strategy，它通过减少 greenhouse gas emissions 来抵消其 own emissions。通过 carbon offsetting，个人或组织可以购买 carbon credits..." + +**Input:** "你能推荐几本好看的科幻小说吗？" +**Output:** "我可以推荐一些好看的 sci-fi 小说。The Three-Body Problem 是一本非常著名的科幻小说..." + +## Limitations + +- **Response length**: Shorter than base Llama (~992 vs 1511 chars). The code-switched training data tends to be shorter. +- **Repetition**: Slightly higher repetition ratio (0.065 vs 0.035). Some responses have repetitive patterns. +- **Helpfulness**: Slightly lower than base (1.64 vs 2.03). The code-switching behavior can reduce response depth. +- **CS rate ≠ 100%**: Despite 99.1% CS rate, a small fraction of responses remain monolingual. +- **Base model knowledge**: Inherits Llama-3.2-3B's knowledge limitations (3B parameter model). + +## Training Infrastructure + +- **Hardware**: 1× H100 GPU +- **Serving**: vLLM for data generation and evaluation +- **LLM Judge**: GLM-5.2 +- **Experiment tracking**: Weights & Biases + +## Related Models + +| Model | Method | CS Rate | Coherence | Description | +|---|---|---|---|---| +| Mercury | GLM full FT | 93.5% | 0.97 | GLM generates responses directly (style leakage) | +| Venus v1 | Base-gen + GLM swap (bug) | 53.7% | 2.16 | EN prompts not translated to ZH first | +| Venus v2 | Base-gen + GLM swap (fixed, 900 ex) | 95.4% | 1.64 | Small-scale proof of concept | +| **Venus v3** | **Base-gen + GLM swap (10K, filtered)** | **99.1%** | **1.96** | **This model — target achieved** | +| Earth v2 | RSF self-distillation (9.5K, 5 ep) | 95.4% | 2.07 | No GLM, Llama teaches itself | + +## Citation + +```bibtex +@misc{venus-v3-2026, + author = {Ziliang Peng}, + title = {Venus v3: Code-Switching Llama-3.2-3B via Response-Level Distillation}, + year = {2026}, + url = {https://huggingface.co/ziliangpeng/llama-3.2-3b-cs-venus-v3} +} +``` + +## License + +This model is governed by the Llama 3 License. See [LICENSE](https://llama.meta.com/llama3/license/). diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..1bad6a0 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,93 @@ +{{- bos_token }} +{%- if custom_tools is defined %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if not tools_in_user_message is defined %} + {%- set tools_in_user_message = true %} +{%- endif %} +{%- if not date_string is defined %} + {%- if strftime_now is defined %} + {%- set date_string = strftime_now("%d %b %Y") %} + {%- else %} + {%- set date_string = "26 Jul 2024" %} + {%- endif %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- set system_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} +{%- else %} + {%- set system_message = "" %} +{%- endif %} + +{#- System message #} +{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} +{%- if tools is not none %} + {{- "Environment: ipython\n" }} +{%- endif %} +{{- "Cutting Knowledge Date: December 2023\n" }} +{{- "Today Date: " + date_string + "\n\n" }} +{%- if tools is not none and not tools_in_user_message %} + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} +{%- endif %} +{{- system_message }} +{{- "<|eot_id|>" }} + +{#- Custom tools are passed in a user message with some extra guidance #} +{%- if tools_in_user_message and not tools is none %} + {#- Extract the first user message so we can plug it in here #} + {%- if messages | length != 0 %} + {%- set first_user_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} + {%- else %} + {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} +{%- endif %} + {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} + {{- "Given the following functions, please respond with a JSON for a function call " }} + {{- "with its proper arguments that best answers the given prompt.\n\n" }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {{- first_user_message + "<|eot_id|>"}} +{%- endif %} + +{%- for message in messages %} + {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} + {%- elif 'tool_calls' in message %} + {%- if not message.tool_calls|length == 1 %} + {{- raise_exception("This model only supports single tool-calls at once!") }} + {%- endif %} + {%- set tool_call = message.tool_calls[0].function %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} + {{- '{"name": "' + tool_call.name + '", ' }} + {{- '"parameters": ' }} + {{- tool_call.arguments | tojson }} + {{- "}" }} + {{- "<|eot_id|>" }} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} + {%- if message.content is mapping or message.content is iterable %} + {{- message.content | tojson }} + {%- else %} + {{- message.content }} + {%- endif %} + {{- "<|eot_id|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..259bcd7 --- /dev/null +++ b/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128009, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 128009, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_parameters": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_theta": 500000.0, + "rope_type": "llama3" + }, + "tie_word_embeddings": true, + "transformers_version": "5.14.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..50143b4 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,14 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128009, + 128001, + 128008, + 128009 + ], + "pad_token_id": 128009, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "5.14.1" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..8035027 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a975f95852b72ce65d0ce71d26e939090c998f415b07c73e19a9b5bd219df081 +size 6425529112 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..02ee80b --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,16 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..a3ccf3a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "backend": "tokenizers", + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|eot_id|>", + "is_local": false, + "local_files_only": false, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|eot_id|>", + "tokenizer_class": "TokenizersBackend" +}