llm-jp/optimal-sparsity-code-d2048-E64-k4-26.4B-A2.3B
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: llm-jp/optimal-sparsity-code-d2048-E64-k4-26.4B-A2.3B
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-04-12 11:52:58 +08:00
commit 4f4cfacd36
19 changed files with 3382 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
.gitattributes vendored Normal file
View File

@@ -0,0 +1,60 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bin.* filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zstandard filter=lfs diff=lfs merge=lfs -text
*.tfevents* filter=lfs diff=lfs merge=lfs -text
*.db* filter=lfs diff=lfs merge=lfs -text
*.ark* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.gguf* filter=lfs diff=lfs merge=lfs -text
*.ggml filter=lfs diff=lfs merge=lfs -text
*.llamafile* filter=lfs diff=lfs merge=lfs -text
*.pt2 filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
model-00004-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text
model-00005-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00009-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00008-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00007-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00002-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00006-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00010-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00011-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00003-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00001-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text

31
README.md Normal file
View File

@@ -0,0 +1,31 @@
---
pipeline_tag: text-generation
library_name: transformers
license: apache-2.0
tags:
- mixtral
- moe
- reasoning
---
# Optimal Sparsity of Mixture-of-Experts Language Models for Reasoning Tasks
This repository contains model checkpoints from the paper [Optimal Sparsity of Mixture-of-Experts Language Models for Reasoning Tasks](https://huggingface.co/papers/2508.18672).
For more details, including code and evaluation procedures, please refer to the official GitHub repository: [https://github.com/rioyokotalab/optimal-sparsity](https://github.com/rioyokotalab/optimal-sparsity)
## How to cite
If you find our work helpful, please feel free to cite the paper.
```bibtex
@inproceedings{
nakamura2026optimal,
title={Optimal Sparsity of Mixture-of-Experts Language Models for Reasoning Tasks},
author={Taishi Nakamura and Satoki Ishikawa and Masaki Kawamura and Takumi Okamoto and Daisuke Nohara and Jun Suzuki and Rio Yokota},
booktitle={The Fourteenth International Conference on Learning Representations},
year={2026},
url={https://openreview.net/forum?id=XFw2EPRUUR}
}
```

32
config.json Normal file
View File

@@ -0,0 +1,32 @@
{
"architectures": [
"MixtralForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"hidden_act": "silu",
"hidden_size": 2048,
"initializer_range": 0.02,
"intermediate_size": 4096,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "mixtral",
"num_attention_heads": 16,
"num_experts_per_tok": 4,
"num_hidden_layers": 16,
"num_key_value_heads": 16,
"num_local_experts": 64,
"output_router_logits": false,
"rms_norm_eps": 1e-05,
"rope_theta": 10000,
"router_aux_loss_coef": 0.01,
"router_jitter_noise": 0.0,
"sliding_window": null,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.46.0",
"use_cache": true,
"vocab_size": 99584
}

1
configuration.json Normal file
View File

@@ -0,0 +1 @@
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}

3
model-00001-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dd862d19352933eb6bf5b4fd37d54a94017000b9a6a5512faf7f73702d4c3100
size 4992445944

3
model-00002-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d40b386bb4c1bc6bae8a3afd1eb14117cd8a51e2767d38a4a334614379e09d4b
size 4999649272

3
model-00003-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a59ddbddd8bb0be28dd454e7f3af646291df9762d4e2f8008a322bca04fe2255
size 4999649400

3
model-00004-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:917ab5f1ca1172883fbdd970b7b399f7ce5b8fb4347191f65e9868a10d05baf9
size 4999649440

3
model-00005-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dabaf4cb4a5724d63f7cbd1a88018c8dab05850007f907623d4e72d7d95eb6e3
size 4999649368

3
model-00006-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a6cddfd752de0980c3f008a9a99c6591c5e731453845d30c7ab13a2ce1661b8c
size 4999649416

3
model-00007-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f574c05e36062f3f1df48443982979cfe5462c9a6343a1cc4ebf6309bb3eb89d
size 4999649456

3
model-00008-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:67be2f4b51fd539fb8b589f0e5fc1dc75e2e76c03ff6e9fb5dd62d9f0fa82a89
size 4999649456

3
model-00009-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:07dbb7bb96696bf2d7b67b061ff0dfb3e3a6b281bc8609524f7b5367d0087786
size 4999649392

3
model-00010-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b9af4b6bfbc8ca26579c1ea60703f2b5f3d129b46d495393d1bf15a068e719f5
size 4999649376

3
model-00011-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:93d81d7e9f8affb1193b994a02b5cbfd5a42c4cbdc35d4b8ea32be7befefdfd3
size 2907725152

3194
model.safetensors.index.json Normal file
View File

File diff suppressed because it is too large Load Diff

10
special_tokens_map.json Normal file
View File

@@ -0,0 +1,10 @@
{
"bos_token": "<s>",
"cls_token": "<CLS|LLM-jp>",
"eod_token": "</s>",
"eos_token": "</s>",
"mask_token": "<MASK|LLM-jp>",
"pad_token": "<PAD|LLM-jp>",
"sep_token": "<SEP|LLM-jp>",
"unk_token": "<unk>"
}

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:955dc1fa623fab38cc92a3f4ee172423ae6d73201c4207569bfdf5626bc733f0
size 6416433

18
tokenizer_config.json Normal file
View File

@@ -0,0 +1,18 @@
{
"add_bos_token": true,
"add_eos_token": false,
"unk_token": "<unk>",
"bos_token": "<s>",
"eos_token": "</s>",
"pad_token": "<PAD|LLM-jp>",
"cls_token": "<CLS|LLM-jp>",
"sep_token": "<SEP|LLM-jp>",
"eod_token": "</s>",
"mask_token": "<MASK|LLM-jp>",
"extra_ids": 0,
"sp_model_kwargs": {},
"model_max_length": 1000000000000000019884624838656,
"clean_up_tokenization_spaces": false,
"special_tokens_map_file": null,
"tokenizer_class": "PreTrainedTokenizerFast"
}