初始化项目，由ModelHub XC社区提供模型
Model: state-spaces/mamba-130m-hf Source: Original Platform
This commit is contained in:
ModelHub XC
35
.gitattributes
vendored
Normal file
35
.gitattributes
vendored
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
76
README.md
Normal file
76
README.md
Normal file
@@ -0,0 +1,76 @@
|
|||||||
|
---
|
||||||
|
library_name: transformers
|
||||||
|
tags: []
|
||||||
|
---
|
||||||
|
|
||||||
|
# Mamba
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
This repository contains the `transfromers` compatible `mamba-2.8b`. The checkpoints are untouched, but the full `config.json` and tokenizer are pushed to this repo.
|
||||||
|
|
||||||
|
# Usage
|
||||||
|
|
||||||
|
You need to install `transformers` from `main` until `transformers=4.39.0` is released.
|
||||||
|
```bash
|
||||||
|
pip install git+https://github.com/huggingface/transformers@main
|
||||||
|
```
|
||||||
|
|
||||||
|
We also recommend you to install both `causal_conv_1d` and `mamba-ssm` using:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pip install causal-conv1d>=1.2.0
|
||||||
|
pip install mamba-ssm
|
||||||
|
```
|
||||||
|
|
||||||
|
If any of these two is not installed, the "eager" implementation will be used. Otherwise the more optimised `cuda` kernels will be used.
|
||||||
|
|
||||||
|
## Generation
|
||||||
|
You can use the classic `generate` API:
|
||||||
|
```python
|
||||||
|
>>> from transformers import MambaConfig, MambaForCausalLM, AutoTokenizer
|
||||||
|
>>> import torch
|
||||||
|
|
||||||
|
>>> tokenizer = AutoTokenizer.from_pretrained("state-spaces/mamba-130m-hf")
|
||||||
|
>>> model = MambaForCausalLM.from_pretrained("state-spaces/mamba-130m-hf")
|
||||||
|
>>> input_ids = tokenizer("Hey how are you doing?", return_tensors="pt")["input_ids"]
|
||||||
|
|
||||||
|
>>> out = model.generate(input_ids, max_new_tokens=10)
|
||||||
|
>>> print(tokenizer.batch_decode(out))
|
||||||
|
["Hey how are you doing?\n\nI'm so glad you're here."]
|
||||||
|
```
|
||||||
|
|
||||||
|
## PEFT finetuning example
|
||||||
|
In order to finetune using the `peft` library, we recommend keeping the model in float32!
|
||||||
|
|
||||||
|
```python
|
||||||
|
from datasets import load_dataset
|
||||||
|
from trl import SFTTrainer
|
||||||
|
from peft import LoraConfig
|
||||||
|
from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained("state-spaces/mamba-130m-hf")
|
||||||
|
model = AutoModelForCausalLM.from_pretrained("state-spaces/mamba-130m-hf")
|
||||||
|
dataset = load_dataset("Abirate/english_quotes", split="train")
|
||||||
|
training_args = TrainingArguments(
|
||||||
|
output_dir="./results",
|
||||||
|
num_train_epochs=3,
|
||||||
|
per_device_train_batch_size=4,
|
||||||
|
logging_dir='./logs',
|
||||||
|
logging_steps=10,
|
||||||
|
learning_rate=2e-3
|
||||||
|
)
|
||||||
|
lora_config = LoraConfig(
|
||||||
|
r=8,
|
||||||
|
target_modules=["x_proj", "embeddings", "in_proj", "out_proj"],
|
||||||
|
task_type="CAUSAL_LM",
|
||||||
|
bias="none"
|
||||||
|
)
|
||||||
|
trainer = SFTTrainer(
|
||||||
|
model=model,
|
||||||
|
tokenizer=tokenizer,
|
||||||
|
args=training_args,
|
||||||
|
peft_config=lora_config,
|
||||||
|
train_dataset=dataset,
|
||||||
|
dataset_text_field="quote",
|
||||||
|
)
|
||||||
|
trainer.train()
|
||||||
|
```
|
||||||
39
config.json
Normal file
39
config.json
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"MambaForCausalLM"
|
||||||
|
],
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"conv_kernel": 4,
|
||||||
|
"d_inner": 1536,
|
||||||
|
"d_model": 768,
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"expand": 2,
|
||||||
|
"fused_add_norm": true,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 768,
|
||||||
|
"initializer_range": 0.1,
|
||||||
|
"intermediate_size": 1536,
|
||||||
|
"layer_norm_epsilon": 1e-05,
|
||||||
|
"model_type": "mamba",
|
||||||
|
"n_layer": 24,
|
||||||
|
"num_hidden_layers": 24,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"pad_vocab_size_multiple": 8,
|
||||||
|
"rescale_prenorm_residual": false,
|
||||||
|
"residual_in_fp32": true,
|
||||||
|
"rms_norm": true,
|
||||||
|
"ssm_cfg": {},
|
||||||
|
"state_size": 16,
|
||||||
|
"time_step_floor": 0.0001,
|
||||||
|
"time_step_init_scheme": "random",
|
||||||
|
"time_step_max": 0.1,
|
||||||
|
"time_step_min": 0.001,
|
||||||
|
"time_step_rank": 48,
|
||||||
|
"time_step_scale": 1.0,
|
||||||
|
"torch_dtype": "float32",
|
||||||
|
"transformers_version": "4.39.0.dev0",
|
||||||
|
"use_bias": false,
|
||||||
|
"use_cache": true,
|
||||||
|
"use_conv_bias": true,
|
||||||
|
"vocab_size": 50280
|
||||||
|
}
|
||||||
7
generation_config.json
Normal file
7
generation_config.json
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"_from_model_config": true,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"transformers_version": "4.39.0.dev0"
|
||||||
|
}
|
||||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:1a5ed29c492ef4d485df3b7c2c8109771696589855b2162ad1ba618b6067cbea
|
||||||
|
size 516567560
|
||||||
100534
tokenizer.json
Normal file
100534
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
212
tokenizer_config.json
Normal file
212
tokenizer_config.json
Normal file
@@ -0,0 +1,212 @@
|
|||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"added_tokens_decoder": {
|
||||||
|
"0": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"1": {
|
||||||
|
"content": "<|padding|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"50254": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50255": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50256": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50257": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50258": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50259": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50260": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50261": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50262": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50263": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50264": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50265": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50266": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50267": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50268": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50269": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50270": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50271": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50272": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50273": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50274": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50275": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"50276": {
|
||||||
|
"content": " ",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": true,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"bos_token": "<|endoftext|>",
|
||||||
|
"clean_up_tokenization_spaces": true,
|
||||||
|
"eos_token": "<|endoftext|>",
|
||||||
|
"model_max_length": 1000000000000000019884624838656,
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"tokenizer_class": "GPTNeoXTokenizer",
|
||||||
|
"unk_token": "<|endoftext|>"
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user