初始化项目，由ModelHub XC社区提供模型
Model: soumil1/mamba2-10m-slimpajama-500m Source: Original Platform
This commit is contained in:
ModelHub XC
35
.gitattributes
vendored
Normal file
35
.gitattributes
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
111
README.md
Normal file
111
README.md
Normal file
@@ -0,0 +1,111 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
library_name: transformers
|
||||
tags:
|
||||
- mamba2
|
||||
- mamba
|
||||
- vllm
|
||||
- state-space-model
|
||||
- causal-language-modeling
|
||||
- slimpajama
|
||||
- hpml
|
||||
datasets:
|
||||
- cerebras/SlimPajama-627B
|
||||
pipeline_tag: text-generation
|
||||
---
|
||||
|
||||
# Mamba2 10M SlimPajama 500M
|
||||
|
||||
This is the vLLM/Transformers-compatible export of the Mamba2 baseline from our HPML Simamba discretization experiments. It is the same checkpoint as `soumil1/mamba2-10m-slimpajama-500m`, converted from the training repository's `mamba_ssm` checkpoint layout to the standard Hugging Face `Mamba2ForCausalLM` layout that vLLM can identify natively.
|
||||
|
||||
The model is a research checkpoint, not an instruction-tuned assistant.
|
||||
|
||||
## Checkpoint
|
||||
|
||||
| Field | Value |
|
||||
| --- | --- |
|
||||
| Variant | Mamba2 baseline |
|
||||
| Parameters | 9,472,400 unique parameters with tied input/output embeddings |
|
||||
| Training data | SlimPajama subset, `data/slimpajama_500m_50m/train.bin` |
|
||||
| Validation data | Held-out SlimPajama subset, `data/slimpajama_500m_50m/val.bin` |
|
||||
| Tokenizer | `EleutherAI/gpt-neox-20b` tokenizer files are included |
|
||||
| Context length | 128 tokens during training |
|
||||
| Best checkpoint step | 122,000 |
|
||||
| Best validation loss | 4.86253547668457 |
|
||||
| W&B run | https://wandb.ai/ssb2234-columbia/simamba/runs/dm74y180 |
|
||||
|
||||
## vLLM
|
||||
|
||||
The repository uses:
|
||||
|
||||
```json
|
||||
{
|
||||
"model_type": "mamba2",
|
||||
"architectures": ["Mamba2ForCausalLM"]
|
||||
}
|
||||
```
|
||||
|
||||
Serve with vLLM:
|
||||
|
||||
```bash
|
||||
vllm serve soumil1/mamba2-10m-slimpajama-500m \
|
||||
--dtype float32 \
|
||||
--max-model-len 128
|
||||
```
|
||||
|
||||
For offline inference:
|
||||
|
||||
```python
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
llm = LLM(
|
||||
model="soumil1/mamba2-10m-slimpajama-500m",
|
||||
dtype="float32",
|
||||
max_model_len=128,
|
||||
)
|
||||
outputs = llm.generate(
|
||||
["State space models are"],
|
||||
SamplingParams(max_tokens=32, temperature=0.8),
|
||||
)
|
||||
print(outputs[0].outputs[0].text)
|
||||
```
|
||||
|
||||
## Transformers
|
||||
|
||||
```python
|
||||
import torch
|
||||
from transformers import AutoTokenizer, Mamba2ForCausalLM
|
||||
|
||||
repo_id = "soumil1/mamba2-10m-slimpajama-500m"
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(repo_id)
|
||||
model = Mamba2ForCausalLM.from_pretrained(repo_id, torch_dtype=torch.float32).cuda()
|
||||
model.eval()
|
||||
|
||||
input_ids = tokenizer("State space models are", return_tensors="pt").input_ids.cuda()
|
||||
with torch.no_grad():
|
||||
logits = model(input_ids).logits
|
||||
print(logits.shape)
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
```json
|
||||
{
|
||||
"hidden_size": 160,
|
||||
"num_hidden_layers": 8,
|
||||
"vocab_size": 50280,
|
||||
"state_size": 64,
|
||||
"conv_kernel": 4,
|
||||
"expand": 2,
|
||||
"head_dim": 32,
|
||||
"num_heads": 10,
|
||||
"n_groups": 1,
|
||||
"chunk_size": 16,
|
||||
"tie_word_embeddings": true
|
||||
}
|
||||
```
|
||||
|
||||
## Limitations
|
||||
|
||||
This is a small pretraining checkpoint trained for controlled architecture comparison. It was trained on a limited SlimPajama subset with a short sequence length, so generation quality and downstream task performance should be interpreted cautiously.
|
||||
8
checkpoint_manifest.json
Normal file
8
checkpoint_manifest.json
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"step": 122001,
|
||||
"kind": "best",
|
||||
"include_optimizer": false,
|
||||
"created_at_unix": 1777829310,
|
||||
"resume_step": 122001,
|
||||
"wandb_id": "dm74y180"
|
||||
}
|
||||
57
config.json
Normal file
57
config.json
Normal file
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"transformers_version": "5.7.0",
|
||||
"architectures": [
|
||||
"Mamba2ForCausalLM"
|
||||
],
|
||||
"output_hidden_states": false,
|
||||
"return_dict": true,
|
||||
"dtype": "float32",
|
||||
"chunk_size_feed_forward": 0,
|
||||
"is_encoder_decoder": false,
|
||||
"id2label": {
|
||||
"0": "LABEL_0",
|
||||
"1": "LABEL_1"
|
||||
},
|
||||
"label2id": {
|
||||
"LABEL_0": 0,
|
||||
"LABEL_1": 1
|
||||
},
|
||||
"problem_type": null,
|
||||
"num_heads": 10,
|
||||
"head_dim": 32,
|
||||
"vocab_size": 50280,
|
||||
"hidden_size": 160,
|
||||
"state_size": 64,
|
||||
"num_hidden_layers": 8,
|
||||
"layer_norm_epsilon": 1e-05,
|
||||
"pad_token_id": 1,
|
||||
"bos_token_id": 0,
|
||||
"eos_token_id": 0,
|
||||
"expand": 2,
|
||||
"conv_kernel": 4,
|
||||
"n_groups": 1,
|
||||
"use_bias": false,
|
||||
"use_conv_bias": true,
|
||||
"hidden_act": "silu",
|
||||
"initializer_range": 0.1,
|
||||
"residual_in_fp32": true,
|
||||
"time_step_rank": 10,
|
||||
"time_step_min": 0.001,
|
||||
"time_step_max": 0.1,
|
||||
"time_step_floor": 0.0001,
|
||||
"time_step_limit": [
|
||||
0.0,
|
||||
Infinity
|
||||
],
|
||||
"rescale_prenorm_residual": false,
|
||||
"use_cache": true,
|
||||
"rms_norm": true,
|
||||
"chunk_size": 16,
|
||||
"tie_word_embeddings": true,
|
||||
"_name_or_path": "",
|
||||
"model_type": "mamba2",
|
||||
"output_attentions": false,
|
||||
"intermediate_size": 320,
|
||||
"norm_before_gate": false,
|
||||
"torch_dtype": "float32"
|
||||
}
|
||||
4
metrics.json
Normal file
4
metrics.json
Normal file
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"step": 122000,
|
||||
"val/loss": 4.86253547668457
|
||||
}
|
||||
3
pytorch_model.bin
Normal file
3
pytorch_model.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:8d28cd55dfa0a96fc4b0493c2bb07064f942bed14918690f59540189e84e1a82
|
||||
size 37912012
|
||||
250557
tokenizer.json
Normal file
250557
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
14
tokenizer_config.json
Normal file
14
tokenizer_config.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"backend": "tokenizers",
|
||||
"bos_token": "<|endoftext|>",
|
||||
"eos_token": "<|endoftext|>",
|
||||
"errors": "replace",
|
||||
"is_local": false,
|
||||
"local_files_only": false,
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"pad_token": "<|padding|>",
|
||||
"tokenizer_class": "GPTNeoXTokenizer",
|
||||
"trim_offsets": true,
|
||||
"unk_token": "<|endoftext|>"
|
||||
}
|
||||
Reference in New Issue
Block a user