初始化项目，由ModelHub XC社区提供模型

Model: soumil1/mamba2-10m-slimpajama-500m
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,111 @@
+---
+license: apache-2.0
+library_name: transformers
+tags:
+  - mamba2
+  - mamba
+  - vllm
+  - state-space-model
+  - causal-language-modeling
+  - slimpajama
+  - hpml
+datasets:
+  - cerebras/SlimPajama-627B
+pipeline_tag: text-generation
+---
+
+# Mamba2 10M SlimPajama 500M
+
+This is the vLLM/Transformers-compatible export of the Mamba2 baseline from our HPML Simamba discretization experiments. It is the same checkpoint as `soumil1/mamba2-10m-slimpajama-500m`, converted from the training repository's `mamba_ssm` checkpoint layout to the standard Hugging Face `Mamba2ForCausalLM` layout that vLLM can identify natively.
+
+The model is a research checkpoint, not an instruction-tuned assistant.
+
+## Checkpoint
+
+| Field | Value |
+| --- | --- |
+| Variant | Mamba2 baseline |
+| Parameters | 9,472,400 unique parameters with tied input/output embeddings |
+| Training data | SlimPajama subset, `data/slimpajama_500m_50m/train.bin` |
+| Validation data | Held-out SlimPajama subset, `data/slimpajama_500m_50m/val.bin` |
+| Tokenizer | `EleutherAI/gpt-neox-20b` tokenizer files are included |
+| Context length | 128 tokens during training |
+| Best checkpoint step | 122,000 |
+| Best validation loss | 4.86253547668457 |
+| W&B run | https://wandb.ai/ssb2234-columbia/simamba/runs/dm74y180 |
+
+## vLLM
+
+The repository uses:
+
+```json
+{
+  "model_type": "mamba2",
+  "architectures": ["Mamba2ForCausalLM"]
+}
+```
+
+Serve with vLLM:
+
+```bash
+vllm serve soumil1/mamba2-10m-slimpajama-500m \
+  --dtype float32 \
+  --max-model-len 128
+```
+
+For offline inference:
+
+```python
+from vllm import LLM, SamplingParams
+
+llm = LLM(
+    model="soumil1/mamba2-10m-slimpajama-500m",
+    dtype="float32",
+    max_model_len=128,
+)
+outputs = llm.generate(
+    ["State space models are"],
+    SamplingParams(max_tokens=32, temperature=0.8),
+)
+print(outputs[0].outputs[0].text)
+```
+
+## Transformers
+
+```python
+import torch
+from transformers import AutoTokenizer, Mamba2ForCausalLM
+
+repo_id = "soumil1/mamba2-10m-slimpajama-500m"
+
+tokenizer = AutoTokenizer.from_pretrained(repo_id)
+model = Mamba2ForCausalLM.from_pretrained(repo_id, torch_dtype=torch.float32).cuda()
+model.eval()
+
+input_ids = tokenizer("State space models are", return_tensors="pt").input_ids.cuda()
+with torch.no_grad():
+    logits = model(input_ids).logits
+print(logits.shape)
+```
+
+## Architecture
+
+```json
+{
+  "hidden_size": 160,
+  "num_hidden_layers": 8,
+  "vocab_size": 50280,
+  "state_size": 64,
+  "conv_kernel": 4,
+  "expand": 2,
+  "head_dim": 32,
+  "num_heads": 10,
+  "n_groups": 1,
+  "chunk_size": 16,
+  "tie_word_embeddings": true
+}
+```
+
+## Limitations
+
+This is a small pretraining checkpoint trained for controlled architecture comparison. It was trained on a limited SlimPajama subset with a short sequence length, so generation quality and downstream task performance should be interpreted cautiously.

checkpoint_manifest.json

@@ -0,0 +1,8 @@
+{
+  "step": 122001,
+  "kind": "best",
+  "include_optimizer": false,
+  "created_at_unix": 1777829310,
+  "resume_step": 122001,
+  "wandb_id": "dm74y180"
+}

config.json

@@ -0,0 +1,57 @@
+{
+  "transformers_version": "5.7.0",
+  "architectures": [
+    "Mamba2ForCausalLM"
+  ],
+  "output_hidden_states": false,
+  "return_dict": true,
+  "dtype": "float32",
+  "chunk_size_feed_forward": 0,
+  "is_encoder_decoder": false,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "problem_type": null,
+  "num_heads": 10,
+  "head_dim": 32,
+  "vocab_size": 50280,
+  "hidden_size": 160,
+  "state_size": 64,
+  "num_hidden_layers": 8,
+  "layer_norm_epsilon": 1e-05,
+  "pad_token_id": 1,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "expand": 2,
+  "conv_kernel": 4,
+  "n_groups": 1,
+  "use_bias": false,
+  "use_conv_bias": true,
+  "hidden_act": "silu",
+  "initializer_range": 0.1,
+  "residual_in_fp32": true,
+  "time_step_rank": 10,
+  "time_step_min": 0.001,
+  "time_step_max": 0.1,
+  "time_step_floor": 0.0001,
+  "time_step_limit": [
+    0.0,
+    Infinity
+  ],
+  "rescale_prenorm_residual": false,
+  "use_cache": true,
+  "rms_norm": true,
+  "chunk_size": 16,
+  "tie_word_embeddings": true,
+  "_name_or_path": "",
+  "model_type": "mamba2",
+  "output_attentions": false,
+  "intermediate_size": 320,
+  "norm_before_gate": false,
+  "torch_dtype": "float32"
+}

metrics.json

@@ -0,0 +1,4 @@
+{
+  "step": 122000,
+  "val/loss": 4.86253547668457
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d28cd55dfa0a96fc4b0493c2bb07064f942bed14918690f59540189e84e1a82
+size 37912012

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|padding|>",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<|endoftext|>"
+}