llm-jp/optimal-sparsity-math-d2048-E64-k16-26.4B-A7.1B
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: llm-jp/optimal-sparsity-math-d2048-E64-k16-26.4B-A7.1B
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-04-13 10:59:36 +08:00
commit 7a70b241fd
19 changed files with 3381 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
.gitattributes vendored Normal file
View File

@@ -0,0 +1,59 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bin.* filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zstandard filter=lfs diff=lfs merge=lfs -text
*.tfevents* filter=lfs diff=lfs merge=lfs -text
*.db* filter=lfs diff=lfs merge=lfs -text
*.ark* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.gguf* filter=lfs diff=lfs merge=lfs -text
*.ggml filter=lfs diff=lfs merge=lfs -text
*.llamafile* filter=lfs diff=lfs merge=lfs -text
*.pt2 filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
model-00001-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00003-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00007-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00008-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00004-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00002-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text
model-00010-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00005-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00011-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00009-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text

31
README.md Normal file
View File

@@ -0,0 +1,31 @@
---
pipeline_tag: text-generation
library_name: transformers
license: apache-2.0
tags:
- mixtral
- moe
- reasoning
---
# Optimal Sparsity of Mixture-of-Experts Language Models for Reasoning Tasks
This repository contains model checkpoints from the paper [Optimal Sparsity of Mixture-of-Experts Language Models for Reasoning Tasks](https://huggingface.co/papers/2508.18672).
For more details, including code and evaluation procedures, please refer to the official GitHub repository: [https://github.com/rioyokotalab/optimal-sparsity](https://github.com/rioyokotalab/optimal-sparsity)
## How to cite
If you find our work helpful, please feel free to cite the paper.
```bibtex
@inproceedings{
nakamura2026optimal,
title={Optimal Sparsity of Mixture-of-Experts Language Models for Reasoning Tasks},
author={Taishi Nakamura and Satoki Ishikawa and Masaki Kawamura and Takumi Okamoto and Daisuke Nohara and Jun Suzuki and Rio Yokota},
booktitle={The Fourteenth International Conference on Learning Representations},
year={2026},
url={https://openreview.net/forum?id=XFw2EPRUUR}
}
```

32
config.json Normal file
View File

@@ -0,0 +1,32 @@
{
"architectures": [
"MixtralForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"hidden_act": "silu",
"hidden_size": 2048,
"initializer_range": 0.02,
"intermediate_size": 4096,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "mixtral",
"num_attention_heads": 16,
"num_experts_per_tok": 16,
"num_hidden_layers": 16,
"num_key_value_heads": 16,
"num_local_experts": 64,
"output_router_logits": false,
"rms_norm_eps": 1e-05,
"rope_theta": 10000,
"router_aux_loss_coef": 0.01,
"router_jitter_noise": 0.0,
"sliding_window": null,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.46.0",
"use_cache": true,
"vocab_size": 99584
}

1
configuration.json Normal file
View File

@@ -0,0 +1 @@
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}

3
model-00001-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a574b169d11335d445fd9632e36b48ff4330b57d53322804103c994765716046
size 4990278800

3
model-00002-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:fe31a54b55b067a6902ffa5dcc5eb79d07943f1327731f747e51060520638da4
size 4999649280

3
model-00003-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:cb73246e22dd47432f30b3c0df010a8e2f7395ca0a587a160799a0fd32f9bc85
size 4999649312

3
model-00004-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:66d1c9f15a2c248dfce7f7cdbc7bd9568606f12200d322e57d07e6ecfb04eba3
size 4999649312

3
model-00005-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0a2d4940d42b35c1b2ed43ffd4ec434c5cf83e323f7cebd95553c47d6bc0b2f9
size 4999649312

3
model-00006-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5e0a78b929c6a619377595c59b8b020fd6c5c57bd4ffb051c8757db15e286c1d
size 4985039080

3
model-00007-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a05e63444aa598152b92be95425b56a7589c031b0c8a2d2d0e2cc2d605a9290a
size 4999649472

3
model-00008-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:8f6886d2cae98e8c0719603f3bddc7e8feafea3c57bc2b36d9e9ce3391c6bb6f
size 4999649544

3
model-00009-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:27bf1cb88e9f68867f4f97fe959344b14f6cfb55155e2d1540620b8a6509551c
size 4999649536

3
model-00010-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:40aa667ee62a1bb21fe8c2117f999201d2db57c8f95801d370f7f7870e550a8f
size 4999649520

3
model-00011-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:080f51503e752aaf1f3e388a8daeed4ea573a0f52e4aa7d3cb3dded1e7fc9a9f
size 2924502504

3194
model.safetensors.index.json Normal file
View File

File diff suppressed because it is too large Load Diff

10
special_tokens_map.json Normal file
View File

@@ -0,0 +1,10 @@
{
"bos_token": "<s>",
"cls_token": "<CLS|LLM-jp>",
"eod_token": "</s>",
"eos_token": "</s>",
"mask_token": "<MASK|LLM-jp>",
"pad_token": "<PAD|LLM-jp>",
"sep_token": "<SEP|LLM-jp>",
"unk_token": "<unk>"
}

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:955dc1fa623fab38cc92a3f4ee172423ae6d73201c4207569bfdf5626bc733f0
size 6416433

18
tokenizer_config.json Normal file
View File

@@ -0,0 +1,18 @@
{
"add_bos_token": true,
"add_eos_token": false,
"unk_token": "<unk>",
"bos_token": "<s>",
"eos_token": "</s>",
"pad_token": "<PAD|LLM-jp>",
"cls_token": "<CLS|LLM-jp>",
"sep_token": "<SEP|LLM-jp>",
"eod_token": "</s>",
"mask_token": "<MASK|LLM-jp>",
"extra_ids": 0,
"sp_model_kwargs": {},
"model_max_length": 1000000000000000019884624838656,
"clean_up_tokenization_spaces": false,
"special_tokens_map_file": null,
"tokenizer_class": "PreTrainedTokenizerFast"
}