llm-jp/optimal-sparsity-math-d2048-E32-k2-13.6B-A1.5B
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: llm-jp/optimal-sparsity-math-d2048-E32-k2-13.6B-A1.5B
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-04-13 07:54:09 +08:00
commit 5c790f7230
14 changed files with 1826 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
.gitattributes vendored Normal file
View File

@@ -0,0 +1,55 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bin.* filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zstandard filter=lfs diff=lfs merge=lfs -text
*.tfevents* filter=lfs diff=lfs merge=lfs -text
*.db* filter=lfs diff=lfs merge=lfs -text
*.ark* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.gguf* filter=lfs diff=lfs merge=lfs -text
*.ggml filter=lfs diff=lfs merge=lfs -text
*.llamafile* filter=lfs diff=lfs merge=lfs -text
*.pt2 filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text
model-00004-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text
model-00001-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text
model-00005-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text
model-00006-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text
model-00003-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text
model-00002-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text

31
README.md Normal file
View File

@@ -0,0 +1,31 @@
---
pipeline_tag: text-generation
library_name: transformers
license: apache-2.0
tags:
- mixtral
- moe
- reasoning
---
# Optimal Sparsity of Mixture-of-Experts Language Models for Reasoning Tasks
This repository contains model checkpoints from the paper [Optimal Sparsity of Mixture-of-Experts Language Models for Reasoning Tasks](https://huggingface.co/papers/2508.18672).
For more details, including code and evaluation procedures, please refer to the official GitHub repository: [https://github.com/rioyokotalab/optimal-sparsity](https://github.com/rioyokotalab/optimal-sparsity)
## How to cite
If you find our work helpful, please feel free to cite the paper.
```bibtex
@inproceedings{
nakamura2026optimal,
title={Optimal Sparsity of Mixture-of-Experts Language Models for Reasoning Tasks},
author={Taishi Nakamura and Satoki Ishikawa and Masaki Kawamura and Takumi Okamoto and Daisuke Nohara and Jun Suzuki and Rio Yokota},
booktitle={The Fourteenth International Conference on Learning Representations},
year={2026},
url={https://openreview.net/forum?id=XFw2EPRUUR}
}
```

32
config.json Normal file
View File

@@ -0,0 +1,32 @@
{
"architectures": [
"MixtralForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"hidden_act": "silu",
"hidden_size": 2048,
"initializer_range": 0.02,
"intermediate_size": 4096,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "mixtral",
"num_attention_heads": 16,
"num_experts_per_tok": 2,
"num_hidden_layers": 16,
"num_key_value_heads": 16,
"num_local_experts": 32,
"output_router_logits": false,
"rms_norm_eps": 1e-05,
"rope_theta": 10000,
"router_aux_loss_coef": 0.01,
"router_jitter_noise": 0.0,
"sliding_window": null,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"transformers_version": "4.46.0",
"use_cache": true,
"vocab_size": 99584
}

1
configuration.json Normal file
View File

@@ -0,0 +1 @@
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}

3
model-00001-of-00006.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4f5f6c8a55132ed9968c619311337c63f7fe02e7f2e171b1b912a677d21f320a
size 4990348960

3
model-00002-of-00006.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ae1af312d740a42a2c2dede9369c4278e37d8bab1ac8825651b64dafad2c3be1
size 4999649224

3
model-00003-of-00006.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:1d3ff98615d7c51cea0b432d07d73a7b21d923b17b350abc16f71020629b8b2e
size 4999649216

3
model-00004-of-00006.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c4e5da83a42fafd8c1e8f2904b6d5a57c2a0bb89020012ae18f8be116b6647e5
size 4999649392

3
model-00005-of-00006.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e6ecdbf99febd5312ea21fc23b053d03fd93bd7838084d1b763d55dc627ba45b
size 4999649520

3
model-00006-of-00006.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7e5530c01497e04c7ed6017af09c29eaad06168087a1ebba7ef769501c2636dd
size 2135967120

1658
model.safetensors.index.json Normal file
View File

File diff suppressed because it is too large Load Diff

10
special_tokens_map.json Normal file
View File

@@ -0,0 +1,10 @@
{
"bos_token": "<s>",
"cls_token": "<CLS|LLM-jp>",
"eod_token": "</s>",
"eos_token": "</s>",
"mask_token": "<MASK|LLM-jp>",
"pad_token": "<PAD|LLM-jp>",
"sep_token": "<SEP|LLM-jp>",
"unk_token": "<unk>"
}

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:955dc1fa623fab38cc92a3f4ee172423ae6d73201c4207569bfdf5626bc733f0
size 6416433

18
tokenizer_config.json Normal file
View File

@@ -0,0 +1,18 @@
{
"add_bos_token": true,
"add_eos_token": false,
"unk_token": "<unk>",
"bos_token": "<s>",
"eos_token": "</s>",
"pad_token": "<PAD|LLM-jp>",
"cls_token": "<CLS|LLM-jp>",
"sep_token": "<SEP|LLM-jp>",
"eod_token": "</s>",
"mask_token": "<MASK|LLM-jp>",
"extra_ids": 0,
"sp_model_kwargs": {},
"model_max_length": 1000000000000000019884624838656,
"clean_up_tokenization_spaces": false,
"special_tokens_map_file": null,
"tokenizer_class": "PreTrainedTokenizerFast"
}