llm-jp/optimal-sparsity-code-d2048-E64-k8-26.4B-A3.9B
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: llm-jp/optimal-sparsity-code-d2048-E64-k8-26.4B-A3.9B
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-04-12 20:40:58 +08:00
commit 8eec7940b9
19 changed files with 3382 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
.gitattributes vendored Normal file
View File

@@ -0,0 +1,60 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bin.* filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zstandard filter=lfs diff=lfs merge=lfs -text
*.tfevents* filter=lfs diff=lfs merge=lfs -text
*.db* filter=lfs diff=lfs merge=lfs -text
*.ark* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.gguf* filter=lfs diff=lfs merge=lfs -text
*.ggml filter=lfs diff=lfs merge=lfs -text
*.llamafile* filter=lfs diff=lfs merge=lfs -text
*.pt2 filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
model-00007-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00006-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00005-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00001-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00008-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00004-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00011-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00010-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00009-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
model-00002-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text
model-00003-of-00011.safetensors filter=lfs diff=lfs merge=lfs -text

31
README.md Normal file
View File

@@ -0,0 +1,31 @@
---
pipeline_tag: text-generation
library_name: transformers
license: apache-2.0
tags:
- mixtral
- moe
- reasoning
---
# Optimal Sparsity of Mixture-of-Experts Language Models for Reasoning Tasks
This repository contains model checkpoints from the paper [Optimal Sparsity of Mixture-of-Experts Language Models for Reasoning Tasks](https://huggingface.co/papers/2508.18672).
For more details, including code and evaluation procedures, please refer to the official GitHub repository: [https://github.com/rioyokotalab/optimal-sparsity](https://github.com/rioyokotalab/optimal-sparsity)
## How to cite
If you find our work helpful, please feel free to cite the paper.
```bibtex
@inproceedings{
nakamura2026optimal,
title={Optimal Sparsity of Mixture-of-Experts Language Models for Reasoning Tasks},
author={Taishi Nakamura and Satoki Ishikawa and Masaki Kawamura and Takumi Okamoto and Daisuke Nohara and Jun Suzuki and Rio Yokota},
booktitle={The Fourteenth International Conference on Learning Representations},
year={2026},
url={https://openreview.net/forum?id=XFw2EPRUUR}
}
```

32
config.json Normal file
View File

@@ -0,0 +1,32 @@
{
"architectures": [
"MixtralForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"hidden_act": "silu",
"hidden_size": 2048,
"initializer_range": 0.02,
"intermediate_size": 4096,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "mixtral",
"num_attention_heads": 16,
"num_experts_per_tok": 8,
"num_hidden_layers": 16,
"num_key_value_heads": 16,
"num_local_experts": 64,
"output_router_logits": false,
"rms_norm_eps": 1e-05,
"rope_theta": 10000,
"router_aux_loss_coef": 0.01,
"router_jitter_noise": 0.0,
"sliding_window": null,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.46.0",
"use_cache": true,
"vocab_size": 99584
}

1
configuration.json Normal file
View File

@@ -0,0 +1 @@
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}

3
model-00001-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:dea0c93d2422285bc3e6ad1f5eedab81e8768c757f2fd7209243163b12f5c5b9
size 4992445944

3
model-00002-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:77e2dff21ebc4b6c6e64ed0a2a49ab19ca0e2a1b1dc81eb2c1246d5b3171c699
size 4999649272

3
model-00003-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a5ef912f8e5178f175f18d7eda8e85b79edf6a6226cdb2bfed1436b236c06d44
size 4999649400

3
model-00004-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2ee39772bfeafb88ad89e8789250428c8e60283a6f877f2d0db752dad1159cc4
size 4999649440

3
model-00005-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a6400cfdf42a586c1f54d016ed42a56c2e145094974a8a7533b29268a48453a9
size 4999649368

3
model-00006-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:206cb5ce68404f3791c291285209fed3a2b2d195da631fde2b71ce2b223d0dc7
size 4999649416

3
model-00007-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b628f973a6d22f0099fad93faf9345494e37c970340785d165ac36eaa57aff1d
size 4999649456

3
model-00008-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:392000c4bdbd8b1ee4eeead67f3dff09668e0aa526ab86acc37c77a8778e9836
size 4999649456

3
model-00009-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ae91191d1f7f18925718c769b41c004b1fcf6bc94d2de61be99c15766d2e9650
size 4999649392

3
model-00010-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c38c7e0df7e9eb30fe6a4cf4e42c20c6da853e85c9abf509100d5139b2982a1c
size 4999649376

3
model-00011-of-00011.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ebe299fd7f4d3b543d9d0a21a2a77916f9cc614f8ec62d35a7472db3e4a920e4
size 2907725152

3194
model.safetensors.index.json Normal file
View File

File diff suppressed because it is too large Load Diff

10
special_tokens_map.json Normal file
View File

@@ -0,0 +1,10 @@
{
"bos_token": "<s>",
"cls_token": "<CLS|LLM-jp>",
"eod_token": "</s>",
"eos_token": "</s>",
"mask_token": "<MASK|LLM-jp>",
"pad_token": "<PAD|LLM-jp>",
"sep_token": "<SEP|LLM-jp>",
"unk_token": "<unk>"
}

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:955dc1fa623fab38cc92a3f4ee172423ae6d73201c4207569bfdf5626bc733f0
size 6416433

18
tokenizer_config.json Normal file
View File

@@ -0,0 +1,18 @@
{
"add_bos_token": true,
"add_eos_token": false,
"unk_token": "<unk>",
"bos_token": "<s>",
"eos_token": "</s>",
"pad_token": "<PAD|LLM-jp>",
"cls_token": "<CLS|LLM-jp>",
"sep_token": "<SEP|LLM-jp>",
"eod_token": "</s>",
"mask_token": "<MASK|LLM-jp>",
"extra_ids": 0,
"sp_model_kwargs": {},
"model_max_length": 1000000000000000019884624838656,
"clean_up_tokenization_spaces": false,
"special_tokens_map_file": null,
"tokenizer_class": "PreTrainedTokenizerFast"
}