commit 8ac10412703c4a7d0bd99c03726c1cb136871522 Author: ModelHub XC Date: Tue Jul 21 03:36:09 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Erland/mini-glm-moe Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..03f8bae --- /dev/null +++ b/README.md @@ -0,0 +1,88 @@ +--- +license: apache-2.0 +tags: + - moe + - glm + - prime-rl + - testing +--- + +# Mini GLM-4 MoE (0.5B) + +A small [GLM-4 MoE](https://huggingface.co/THUDM/GLM-4-100B-A10B) model (543M parameters) for testing and development. Uses the same `Glm4MoeForCausalLM` architecture as the full GLM-4-100B-A10B but with reduced dimensions. + +This model is designed for testing MoE training pipelines in [prime-rl](https://github.com/PrimeIntellect-ai/prime-rl) without needing large pretrained checkpoints. It is small enough to run on a single GPU while exercising the same code paths as production models. + +## Architecture + +| Parameter | Value | +|---|---| +| Parameters | 543M | +| Hidden size | 1024 | +| Layers | 24 | +| Attention heads | 16 (4 KV heads) | +| Routed experts | 8 | +| Experts per token | 4 | +| Shared experts | 1 | +| MoE intermediate size | 256 | +| Dense intermediate size | 2048 | +| Dense layers (first-k) | 1 | +| Vocab size | 151,552 | +| Partial rotary factor | 0.5 | +| Model type | `glm4_moe` | + +The architecture mirrors [THUDM/GLM-4-100B-A10B](https://huggingface.co/THUDM/GLM-4-100B-A10B): the first layer is a dense MLP, and all subsequent layers use Mixture-of-Experts with a shared expert. Attention uses Grouped Query Attention (GQA) with partial rotary embeddings. + +## How this model was created + +**Step 1: Random initialization.** A `Glm4MoeConfig` was instantiated with the small dimensions above and the HuggingFace `Glm4MoeForCausalLM` model was initialized with random weights. The tokenizer was copied from [THUDM/GLM-4-9B-0414](https://huggingface.co/THUDM/GLM-4-9B-0414). + +**Step 2: Roundtrip verification.** Before training, we verified that the HuggingFace and [prime-rl](https://github.com/PrimeIntellect-ai/prime-rl) custom implementations produce identical outputs on the same weights (max logits diff < 0.01), and that `convert_to_hf` / `convert_to_prime` state dict conversions are lossless. + +**Step 3: SFT warmup.** The model was fine-tuned for 200 steps on [PrimeIntellect/Reverse-Text-SFT](https://huggingface.co/datasets/PrimeIntellect/Reverse-Text-SFT) using prime-rl's custom MoE implementation with the following config: + +```toml +max_steps = 200 + +[model] +impl = "custom" +attn = "sdpa" + +[data] +name = "PrimeIntellect/Reverse-Text-SFT" +batch_size = 4 +seq_len = 1024 + +[optim] +lr = 1e-4 +``` + +Loss went from ~12 (random init) to ~2.5 after 200 steps. The model is not intended to be useful for generation -- the SFT warmup gives it a non-trivial learned distribution so that KL divergence and other RL metrics are meaningful during testing. + +**Step 4: Post-training verification.** After SFT, we re-verified the HF <-> PrimeRL roundtrip on the trained checkpoint to confirm that checkpoint saving (which goes through `convert_to_hf`) produced valid weights. + +## Reproduction + +The scripts used to create this model live in the [prime-rl](https://github.com/PrimeIntellect-ai/prime-rl) repository under `scripts/mini_moe/`: + +```bash +# Step 1: Create random-init model +uv run python scripts/mini_moe/create.py --arch glm4_moe --output-dir ./mini-glm-moe + +# Step 2: Verify HF <-> PrimeRL roundtrip +uv run python scripts/mini_moe/verify.py --arch glm4_moe --model-dir ./mini-glm-moe + +# Step 3: SFT warmup + verify + push +uv run python scripts/mini_moe/sft_warmup.py --arch glm4_moe --model-dir ./mini-glm-moe --sft-steps 200 --push-to-hub samsja/mini-glm-moe +``` + +To add a new architecture, add a preset to `scripts/mini_moe/presets.py`. + +## Intended use + +- Testing MoE training pipelines (SFT, RL) in prime-rl +- Validating state dict conversion between HuggingFace and prime-rl formats +- Integration tests that need a real MoE model but cannot afford large checkpoints +- Checking RL metrics (KL divergence, reward signals) on a small scale + +This model is **not** intended for inference or any downstream task. diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..fdd5837 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,41 @@ +[gMASK] +{%- if tools -%} +<|system|> +# 可用工具 +{% for tool in tools %} + {%- set function = tool.function if tool.get("function") else tool %} + +## {{ function.name }} + +{{ function | tojson(indent=4, ensure_ascii=False) }} +在调用上述函数时，请使用 Json 格式表示调用的参数。 +{%- endfor %} +{%- endif -%} + +{%- for msg in messages %} + {%- if msg.role == 'system' %} +<|system|> +{{ msg.content }} + {%- endif %} +{%- endfor %} + +{%- for message in messages if message.role != 'system' %} + {%- set role = message['role'] %} + {%- set content = message['content'] %} + {%- set meta = message.get("metadata", "") %} + + {%- if role == 'user' %} +<|user|> +{{ content }} + {%- elif role == 'assistant' and not meta %} +<|assistant|> +{{ content }} + {%- elif role == 'assistant' and meta %} +<|assistant|>{{ meta }} +{{ content }} + {%- elif role == 'observation' %} +<|observation|> +{{ content }} + {%- endif %} +{%- endfor %} +{% if add_generation_prompt %}<|assistant|>{% endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..2973e0a --- /dev/null +++ b/config.json @@ -0,0 +1,43 @@ +{ + "architectures": [ + "Glm4MoeForCausalLM" + ], + "attention_bias": true, + "attention_dropout": 0.0, + "dtype": "float32", + "eos_token_id": [ + 151329, + 151336, + 151338 + ], + "first_k_dense_replace": 1, + "hidden_act": "silu", + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 2048, + "max_position_embeddings": 131072, + "model_type": "glm4_moe", + "moe_intermediate_size": 256, + "n_group": 1, + "n_routed_experts": 8, + "n_shared_experts": 1, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_experts_per_tok": 4, + "num_hidden_layers": 24, + "num_key_value_heads": 4, + "pad_token_id": 151329, + "partial_rotary_factor": 0.5, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 1000000, + "routed_scaling_factor": 1.0, + "tie_word_embeddings": false, + "topk_group": 1, + "transformers_version": "4.57.6", + "use_cache": false, + "use_grouped_mm": true, + "use_qk_norm": false, + "vocab_size": 151552, + "head_dim": 64 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..db36174 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,11 @@ +{ + "_from_model_config": true, + "eos_token_id": [ + 151329, + 151336, + 151338 + ], + "pad_token_id": 151329, + "transformers_version": "4.57.6", + "use_cache": false +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..ea2db77 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe548a8bcc13d70ea473a1647770c69acf698dd045c2581ca2907c0cdd03faac +size 1085409024 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..2f85ccd --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,26 @@ +{ + "additional_special_tokens": [ + "<|endoftext|>", + "[MASK]", + "[gMASK]", + "[sMASK]", + "", + "", + "<|system|>", + "<|user|>", + "<|assistant|>", + "<|observation|>", + "<|begin_of_image|>", + "<|end_of_image|>", + "<|begin_of_video|>", + "<|end_of_video|>" + ], + "eos_token": { + "content": "<|user|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|user|>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..4d1dde3 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76ebeac0d8bd7879ead7b43c16b44981f277e47225de2bd7de9ae1a6cc664a8c +size 19966496 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..f43f74c --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,145 @@ +{ + "added_tokens_decoder": { + "151329": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151330": { + "content": "[MASK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151331": { + "content": "[gMASK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151332": { + "content": "[sMASK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151333": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151334": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151335": { + "content": "<|system|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151336": { + "content": "<|user|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151337": { + "content": "<|assistant|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151338": { + "content": "<|observation|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151339": { + "content": "<|begin_of_image|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151340": { + "content": "<|end_of_image|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151341": { + "content": "<|begin_of_video|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151342": { + "content": "<|end_of_video|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|endoftext|>", + "[MASK]", + "[gMASK]", + "[sMASK]", + "", + "", + "<|system|>", + "<|user|>", + "<|assistant|>", + "<|observation|>", + "<|begin_of_image|>", + "<|end_of_image|>", + "<|begin_of_video|>", + "<|end_of_video|>" + ], + "clean_up_tokenization_spaces": false, + "do_lower_case": false, + "eos_token": "<|user|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 128000, + "pad_token": "<|user|>", + "padding_side": "left", + "remove_space": false, + "tokenizer_class": "PreTrainedTokenizerFast" +}