commit abc6e7ed8a97ddf3a75547a4e36e1f3df0ed9b08 Author: ModelHub XC Date: Thu Sep 10 10:44:16 2026 +0800 初始化项目,由ModelHub XC社区提供模型 Model: DALabCommunity/Haidass-143M-v1 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..74bed4a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +logo.png filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..41d794d --- /dev/null +++ b/README.md @@ -0,0 +1,136 @@ +--- +language: +- en +- zh +license: apache-2.0 +datasets: +- openbmb/Ultra-FineWeb +- mlfoundations/dclm-baseline-1.0 +- HuggingFaceTB/finemath +tags: +- haidass +- npu +- bilingual +- mindspeed-llm +library_name: transformers +pipeline_tag: text-generation +--- + +
+ +
+ +# Haidass-143M + +

+English | +中文 +

+ +A bilingual (English/Chinese) small language model trained entirely on **Huawei Ascend** NPU ecosystem. + +## Model Overview + +Haidass-143M is a 143M-parameter bilingual language model trained on approximately 100B tokens of English and Chinese data. The entire training pipeline runs on the Huawei Ascend ecosystem, using the **MindSpeed-LLM** framework on Atlas A2 servers (910B). A custom 64,000-token bilingual vocabulary (SentencePiece BPE) was trained alongside the model. This model is competitive among multilingual models under 150M parameters and ranks favorably across multiple evaluation benchmarks. + +## Model Architecture + +| Parameter | Value | +|------|------| +| Architecture | Qwen3 | +| Layers | 30 | +| Hidden size | 576 | +| Attention heads | 9 | +| KV heads (GQA) | 3 | +| Head dim | 64 | +| FFN intermediate size | 1,536 | +| Vocabulary size | 64,000 | +| Max sequence length | 4,096 | +| Tie word embeddings | Yes | +| Position encoding | RoPE (θ=100,000) | +| Attention bias | None | +| Precision | BF16 | +| Total parameters | ~143M | + +## Training Data + +The model was trained on approximately 100B tokens of mixed English and Chinese data. Primary data sources: + +- [openbmb/Ultra-FineWeb](https://huggingface.co/datasets/openbmb/Ultra-FineWeb) (ultrafineweb-en + ultrafineweb-zh) +- [mlfoundations/dclm-baseline-1.0-parquet](https://huggingface.co/datasets/mlfoundations/dclm-baseline-1.0-parquet) (dclm) +- [HuggingFaceTB/finemath](https://huggingface.co/datasets/HuggingFaceTB/finemath) (finemath-4plus) + + +## Training Configuration + +| Parameter | Value | +|------|------| +| Framework | MindSpeed-LLM (v2.3.0) | +| Hardware | 8 × Atlas A2 servers (8 NPUs per node, 256 cores) | +| NPU model | Huawei Ascend 910B | +| Total NPUs | 64 (8 nodes × 8 cards) | +| Sequence length | 4,096 | + + +## Optimizer + +| Parameter | Value | +|------|------| +| Optimizer | AdamW | +| Peak learning rate | 3e-4 | +| Min learning rate | 3e-5 | + + +## Tokenizer + +| Property | Value | +|------|------| +| Type | SentencePiece BPE | +| Vocabulary size | 64,000 | +| Language coverage | English + Chinese | + +## Evaluation + +Evaluated at checkpoint (~98B tokens) using the lighteval framework (v0.9.2). + +| Benchmark | Score | +|------|------| +| ARC-Easy | 60.44 | +| ARC-Challenge | 27.13 | +| PIQA | 67.25 | +| HellaSwag |37.91 | +| OpenBookQA | 31.8 | +| Winogrande | 52.17 | +| agi_eval | 23.78 | + +## Key Features + +- **Fully Ascend-native**: Trained entirely on Huawei Ascend 910B NPUs using the MindSpeed-LLM framework +- **Bilingual**: Trained on a mixture of English and Chinese data + + +## Intended Use + +This is a research model, suitable for: +- Studying training dynamics of small models on Ascend NPUs +- English/Chinese language modeling research +- Serving as a base model for fine-tuning or annealing experiments + +## Limitations + +- Small model scale; reasoning and generation capabilities are limited +- raw pretrained model only + +## Citation + +```bibtex +@misc{haidass-143m, + title={haidass-143M: A Bilingual Small Language Model Trained on Ascend 910B}, + year={2026}, + note={Based on Qwen3 architecture, trained from scratch on 100B tokens using MindSpeed-LLM on 64× Ascend 910B NPUs} +} +``` + +## License + +Apache 2.0 \ No newline at end of file diff --git a/README_ZH.md b/README_ZH.md new file mode 100644 index 0000000..16400db --- /dev/null +++ b/README_ZH.md @@ -0,0 +1,120 @@ + + +
+ +
+ +# Haidass-143M + +

+English | +中文 +

+ +中英双语小语言模型,在**华为昇腾**生态上进行全流程训练。 + +## 模型简介 + +Haidass-143M 是一个 143M 参数的中英双语语言模型,在约 100B token 的中英文数据上训练完成。模型在华为昇腾生态上进行全流程训练,整体流程基于 **MindSpeed-LLM** 框架和 Atlas A2 服务器(910B)。同时配套训练了大小为 64,000 的中英双语词表。该模型在 150M 以下参数规模的多语言模型中具有较强竞争力,并在多个评测指标中排名靠前。 + +## 模型架构 + +| 参数 | 值 | +|------|-----| +| 架构 | Qwen3 | +| 层数 | 30 | +| 隐层维度 | 576 | +| 注意力头数 | 9 | +| KV 头数 (GQA) | 3 | +| 头维度 | 64 | +| FFN 中间维度 | 1,536 | +| 词表大小 | 64,000 | +| 最大序列长度 | 4,096 | +| 绑定嵌入 | 是 | +| 位置编码 | RoPE (θ=100,000) | +| 注意力偏置 | 无 | +| 精度 | BF16 | +| 总参数量 | ~143M | + +## 训练数据 + +模型在约 100B token 的中英文混合数据上训练。主要数据来源为: + +- [openbmb/Ultra-FineWeb](https://huggingface.co/datasets/openbmb/Ultra-FineWeb) (ultrafineweb-en + ultrafineweb-zh) +- [mlfoundations/dclm-baseline-1.0-parquet](https://huggingface.co/datasets/mlfoundations/dclm-baseline-1.0-parquet) (dclm) +- [HuggingFaceTB/finemath](https://huggingface.co/datasets/HuggingFaceTB/finemath) (finemath-4plus) + + +## 训练配置 + +| 参数 | 值 | +|------|------| +| 框架 | MindSpeed-LLM (v2.3.0) | +| 硬件 | 8 台 Atlas A2 服务器 (每台 8 卡 NPU,256 核) | +| NPU 型号 | 华为昇腾 910B | +| 总 NPU 数 | 64 (8 节点 × 8 卡) | +| 序列长度 | 4,096 | + + +## 优化器 + +| 参数 | 值 | +|------|------| +| 优化器 | AdamW | +| 峰值学习率 | 3e-4 | +| 最低学习率 | 3e-5 | + + +## 词表 + +| 属性 | 值 | +|------|------| +| 类型 | SentencePiece BPE | +| 词表大小 | 64,000 | +| 语言覆盖 | 英文 + 中文 | + +## 测评 + +在 checkpoint (~98B tokens) 上基于 lighteval 框架(v0.9.2)测评。 + +| Benchmark | Score | +|------|------| +| ARC-Easy | 60.44 | +| ARC-Challenge | 27.13 | +| PIQA | 67.25 | +| HellaSwag | 37.91 | +| OpenBookQA | 31.8 | +| Winogrande | 52.17 | +| agi_eval | 23.78 | + +## 核心特点 + +- **全昇腾原生**: 完全在华为昇腾 910B NPU 上训练,使用 MindSpeed-LLM 框架 +- **中英双语**: 模型基于中英混合数据集训练 + + +## 预期用途 + +本模型为研究型模型,适用于: +- 研究小模型在昇腾 NPU 上的训练动态 +- 中英文语言建模研究 +- 作为后续微调或退火实验的基础模型 + +## 局限性 + +- 模型规模较小,推理和生成能力有限 +- 仅为原始预训练模型 + +## Citation + +```bibtex +@misc{haidass-143m, + title={haidass-143M: A Bilingual Small Language Model Trained on Ascend 910B}, + year={2026}, + note={Based on Qwen3 architecture, trained from scratch on 100B tokens using MindSpeed-LLM on 64× Ascend 910B NPUs} +} +``` + +## License + +Apache 2.0 diff --git a/config.json b/config.json new file mode 100644 index 0000000..9d2ae63 --- /dev/null +++ b/config.json @@ -0,0 +1,30 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 3, + "eos_token_id": 5, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 576, + "initializer_range": 0.02, + "intermediate_size": 1536, + "max_position_embeddings": 4096, + "max_window_layers": 30, + "model_type": "qwen3", + "num_attention_heads": 9, + "num_hidden_layers": 30, + "num_key_value_heads": 3, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 100000, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 64000 +} diff --git a/logo.png b/logo.png new file mode 100644 index 0000000..b96356b --- /dev/null +++ b/logo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73961221a08d6f366fafc40ad54d7ecd28e8dd76606e2cc46b2b28b13e71cfc3 +size 688369 diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..f831830 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c28738b1c14dc77891cd81160e7eec8d5b7b20f3f252364ce04bc642c20eefa2 +size 286179680 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..5e9292e --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,33 @@ +{ + "bos_token": null, + "eos_token": "<|im_end|>", + "pad_token": "<|im_end|>", + "unk_token": null, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "", + "", + "<|fim_prefix|>", + "<|fim_middle|>", + "<|fim_suffix|>", + "<|fim_pad|>", + "<|repo_name|>", + "<|file_sep|>", + "", + "", + "", + "" + ] +} \ No newline at end of file diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..1c47e44 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dddb2e3e527f3ff79f3ee1e9c9282001c8c0505ef53107ba75126a26f9318c61 +size 1298093 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e13ad79 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,242 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "3": { + "content": "<|endoftext|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "4": { + "content": "<|im_start|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "5": { + "content": "<|im_end|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "6": { + "content": "<|object_ref_start|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "7": { + "content": "<|object_ref_end|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "8": { + "content": "<|box_start|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "9": { + "content": "<|box_end|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "10": { + "content": "<|quad_start|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "11": { + "content": "<|quad_end|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "12": { + "content": "<|vision_start|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "13": { + "content": "<|vision_end|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "14": { + "content": "<|vision_pad|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "15": { + "content": "<|image_pad|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "16": { + "content": "<|video_pad|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": true + }, + "17": { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": false + }, + "19": { + "content": "<|fim_prefix|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": false + }, + "20": { + "content": "<|fim_middle|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": false + }, + "21": { + "content": "<|fim_suffix|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": false + }, + "22": { + "content": "<|fim_pad|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": false + }, + "23": { + "content": "<|repo_name|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": false + }, + "24": { + "content": "<|file_sep|>", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "rstrip": false, + "normalized": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in content %}\n {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- set content = content.split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content.strip('\\n') + '\\n\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "model_max_length": 131072, + "pad_token": "<|im_end|>", + "split_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": null, + "legacy": true, + "add_eos_token": false, + "sp_model_kwargs": {} +} \ No newline at end of file