commit bbbb68f0566fc5cb7bf7dade70b212eca9ab7a66 Author: ModelHub XC Date: Sat Sep 5 18:06:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: self-long/SelfLong-Llama3.2-3B-Instruct-1M Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..386dec2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,38 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer_config.json filter=lfs diff=lfs merge=lfs -text +model.safetensors.index.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..b75f808 --- /dev/null +++ b/README.md @@ -0,0 +1,92 @@ +--- +license: mit +--- + +# SelfLong-Llama3.2-3B-Instruct-1M + +Wang, Liang, Nan Yang, Xingxing Zhang, Xiaolong Huang, and Furu Wei. "[Bootstrap Your Own Context Length.](https://arxiv.org/pdf/2412.18860)" arXiv preprint arXiv:2412.18860 (2024). + +## Overview + +The SelfLong series of Large Language Models (LLMs) are designed to handle extremely long contexts, reaching up to 1 million tokens. These models, with parameter sizes of 1B, 3B, and 8B, are initialized from the Llama-3.2 and Llama-3.1 architectures. + +## Performance (RULER-1M) + +The following table presents the results of the SelfLong models on the [RULER-1M benchmark](https://huggingface.co/datasets/self-long/RULER-llama3-1M). The numbers represent the RULER score averaged over 13 tasks at different support lengths. + +| Model | Support Length | 32k | 64k | 128k | 256k | 512k | 1M | +|:----------------------| :------------- | :--- | :--- | :--- | :--- | :--- | :--- | +| Llama-3.2-1B-Instruct | 128k | 64.7 | 43.1 | 0.0 | - | - | - | +| Llama-3.2-3B-Instruct | 128k | 77.8 | 70.4 | 0.8 | - | - | - | +| Llama-3.1-8B-Instruct | 128k | **89.8** | **85.4** | 78.5 | - | - | - | +| gradientai/Llama-3-8B-Instruct-Gradient-1048k | 1M | 81.8 | 78.6 | 77.2 | 74.2 | 70.3 | 64.3 | +| **SelfLong-1B-1M** | 1M | 61.3 | 56.6 | 54.7 | 46.7 | 40.7 | 31.1 | +| **SelfLong-3B-1M** | 1M | 80.5 | 78.0 | 75.5 | 68.8 | 58.5 | 38.8 | +| **SelfLong-8B-1M** | 1M | 89.5 | 84.0 | **82.0** | **79.7** | **78.2** | **69.6** | + +**Note:** + +* **Bold** indicates the best performance. +* Underline indicates the second-best performance. +* `-` indicates that the model does not support the given context length. + +## Evaluation on RULER-1M Dataset + +To evaluate the SelfLong models on the RULER-1M dataset, you can follow these steps: + +1. Start vllm server: + +```bash +PROC_PER_NODE=$(nvidia-smi --list-gpus | wc -l) +# Reduce this number if you have limited GPU memory +MAX_MODEL_LEN=1048576 +MODEL_NAME_OR_PATH="self-long/SelfLong-Llama3.2-3B-Instruct-1M" + +echo "Starting VLLM server..." +vllm serve "${MODEL_NAME_OR_PATH}" \ + --dtype auto \ + --disable-log-stats --disable-log-requests --disable-custom-all-reduce \ + --enable_chunked_prefill --max_num_batched_tokens 8192 \ + --tensor-parallel-size "${PROC_PER_NODE}" \ + --max-model-len "${MAX_MODEL_LEN}" \ + --gpu_memory_utilization 0.9 \ + --api-key token-123 & +``` + +2. Get Completions +```python +from openai import OpenAI +from datasets import load_dataset + +client = OpenAI( + base_url="http://localhost:8000/v1", # Default vLLM server address + api_key="token-123" +) + +ds = load_dataset('self-long/RULER-llama3-1M', f'niah_single_1_4k', split='validation') +prompt = ds[0]['input'] + +completion = client.completions.create( + model='self-long/SelfLong-Llama3.2-3B-Instruct-1M', + prompt=prompt, + max_tokens=100, +) + +print(prompt) +print(completion.choices[0].text) +``` + +3. For evaluation, please refer to the evaluation script provided in the RULER repository: [https://github.com/NVIDIA/RULER/blob/main/scripts/eval/evaluate.py](https://github.com/NVIDIA/RULER/blob/main/scripts/eval/evaluate.py). + +Note that different vLLM and Torch versions may produce slightly different decoding results. + +## References + +``` +@article{wang2024bootstrap, + title={Bootstrap Your Own Context Length}, + author={Wang, Liang and Yang, Nan and Zhang, Xingxing and Huang, Xiaolong and Wei, Furu}, + journal={arXiv preprint arXiv:2412.18860}, + year={2024} +} +``` diff --git a/config.json b/config.json new file mode 100644 index 0000000..457ae8a --- /dev/null +++ b/config.json @@ -0,0 +1,41 @@ +{ + "_name_or_path": "selflong-3b/", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 1048576, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 32000000.0, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.49.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..70cd997 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": 128001, + "transformers_version": "4.49.0" +} diff --git a/model-00001-of-00002.safetensors b/model-00001-of-00002.safetensors new file mode 100644 index 0000000..8a2dbd6 --- /dev/null +++ b/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dbd07f0db63fb7b7a1c196d72d21a59b75ee5d12bc656bae2caa389ef1a9db3 +size 4965799096 diff --git a/model-00002-of-00002.safetensors b/model-00002-of-00002.safetensors new file mode 100644 index 0000000..58e3699 --- /dev/null +++ b/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e56e0f69d6ace70db8610fff58601c1833a49d81eeb1fc2ee2b8fa01f3e136b8 +size 1459729952 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..862781f --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a57c30afc07c81610ae7fe013bea3b9d50365f9a8145ae98e8714220f3d7c7c +size 20919 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..344c826 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..36c846e --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9f7acf93ef85fd87b71abd768c58bd07674c6e1063bb08472f25d570350575a +size 54584