commit c736e80fa1e9a2e2f36869a7bc699e1af6f854e5 Author: ModelHub XC Date: Mon Jun 8 23:28:12 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: stepfun-ai/RLVR-8B-0926 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2148334 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,57 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text + +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +model-00002.safetensors filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +figure/train_reward_response_length_1130.png filter=lfs diff=lfs merge=lfs -text +model-00004.safetensors filter=lfs diff=lfs merge=lfs -text +model-00001.safetensors filter=lfs diff=lfs merge=lfs -text +model-00003.safetensors filter=lfs diff=lfs merge=lfs -text +vocab.json filter=lfs diff=lfs merge=lfs -text +merges.txt filter=lfs diff=lfs merge=lfs -text +model-00005.safetensors filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f63be2b --- /dev/null +++ b/README.md @@ -0,0 +1,185 @@ +--- +license: mit +library_name: transformers +pipeline_tag: text-generation +base_model: Qwen/Qwen3-8B-Base +tags: +- reasoning +- test-time-compute +- pacore +- math +- code +--- + +# PaCoRe: Learning to Scale Test-Time Compute with Parallel Coordinated Reasoning + +
+ +[**Read the Paper**](https://arxiv.org/abs/2601.05593) | [**Download Models**](https://huggingface.co/stepfun-ai/PaCoRe-8B) | [**Training Data**](https://huggingface.co/datasets/stepfun-ai/PaCoRe-Train-8k) | [**GitHub**](https://github.com/stepfun-ai/PaCoRe) + +
+ +## 📖 Overview + +We introduce **PaCoRe (Parallel Coordinated Reasoning)**, a framework that shifts the driver of inference from sequential depth to **coordinated parallel breadth**, breaking the model context limitation and massively scaling test time compute: +* **Think in Parallel:** PaCoRe launches massive parallel exploration trajectories. +* **Coordinate in Multi-rounds:** It employs a message-passing architecture to compact these thoughts into concise messages and synthesize them to guide the next round. + +Trained via large-scale, outcome-based reinforcement learning, PaCoRe masters the **Reasoning Synthesis** capabilities required to reconcile diverse parallel insights. + +The approach yields strong improvements across diverse domains, and notably pushes reasoning beyond frontier systems in mathematics: an 8B model reaches 94.5% on HMMT 2025, surpassing GPT-5’s 93.2% by scaling effective TTC to roughly two million tokens. + +We open-source model checkpoints, training data, and the full inference pipeline to accelerate follow-up work! + +------ + +

+ + +

+ +*Figure 1 | Parallel Coordinated Reasoning (PaCoRe) performance. Left: On HMMT 2025, PaCoRe-8B demonstrates remarkable test-time scaling, yielding steady gains and ultimately surpassing GPT-5. Right: On LiveCodeBench, the RLVR-8B model fails to leverage increased test-time compute, while PaCoRe-8B model effectively unlocks substantial gains as the test-time compute increases.* + +

+ + +

+ +*Figure 2 | PaCoRe Training dynamics. Left panels: The Training Reward and Response Length steadily increase, demonstrating the training stability and effectiveness. Right panels: Evaluation on HMMT 2025 and LiveCodeBench (2408-2505). Performance is reported using single round coordinated reasoning in PaCoRe inference setting with $\vec{K} = [16]$.* + +## 🔥 Releases + +**[2025/12/09]** We are excited to release the **PaCoRe-8B** ecosystem: + +* 📝 **In-depth Technical Report:** [**PaCoRe: Learning to Scale Test-Time Compute with Parallel Coordinated Reasoning.**](https://arxiv.org/abs/2601.05593) +* 🤖 **Model:** + * [PaCoRe-8B](https://huggingface.co/stepfun-ai/PaCoRe-8B): Our final PaCoRe-trained model checkpoint! + * [RLVR-8B-0926](https://huggingface.co/stepfun-ai/RLVR-8B-0926): The initial checkpoint of our study, conducted strong reasoning-oriented post-trained on [Qwen3-8B-Base](https://huggingface.co/Qwen/Qwen3-8B-Base). +* 📚 **Data:** [PaCoRe-Train-8k](https://huggingface.co/datasets/stepfun-ai/PaCoRe-Train-8k) The high-quality training corpus, including `opensource_math`, `public_mathcontest`, `synthetic_math` and `code`: + * 🤗 Stage1-3k: [PaCoRe-Train-Stage1-3k](https://huggingface.co/datasets/stepfun-ai/PaCoRe-Train-8k/stage1) + * 🤗 Stage2-5k: [PaCoRe-Train-Stage2-5k](https://huggingface.co/datasets/stepfun-ai/PaCoRe-Train-8k/stage2) + +## 🔍 Experiments + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
HMMT 2025LiveCodeBench (2408-2505)HLEtextMultiChallenge
GPT-593.2 (16k)83.5 (13k)26.0 (14k)71.1 (5.0k)
Qwen3-235B-Thinking82.3 (32k)74.5 (21k)18.2 (23k)60.3 (1.6k)
GLM-4.688.7 (25k)79.5 (19k)17.2 (21k)54.9 (2.2k)
DeepSeek-v3.1-Terminus86.1 (20k)74.9 (11k)19.3 (18k)54.4 (1.1k)
Kimi-K2-Thinking86.5 (33k)79.2 (25k)23.9 (29k)66.4 (1.7k)
RLVR-8B75.4 (48k)70.6 (34k)9.3 (35k)33.3 (1.7k)
PaCoRe-8B (low)88.2 (243k)75.8 (188k)13.0 (196k)41.8 (13k)
PaCoRe-8B (medium)92.9 (869k)76.7 (659k)14.6 (694k)45.7 (45k)
PaCoRe-8B (high)94.5 (1796k)78.2 (1391k)16.2 (1451k)47.0 (95k)
+ +*Table 1 | For each benchmark, we report accuracy together with total TTC (in thousands). For Low, Medium, and High, we apply the inference trajectory configuration as $\vec{K}=[4]$, $[16]$, and $[32, 4]$ separately.* + +### Key Findings +* **Message Passing Unlocks Scaling.** Without compaction, performance flatlines at the context limit. PaCoRe breaks the memory barrier and lets reasoning scale freely. +* **Breadth > Depth.** All compute is not equal. Coordinated parallel reasoning delivers far higher returns than extending a single chain. +* **Data as a Force Multiplier.** The PaCoRe corpus provides exceptionally valuable supervision—even baseline models see substantial gains when trained on it. + +## Getting Started 🚀 +### Installation +First, install the package from the official repository: +```bash +pip install -e . +``` + +### Model Serving +You can directly use `vllm serve` to serve the model: +```bash +vllm serve stepfun-ai/PaCoRe-8B +``` + +### Inference Example +Next, you can run our example inference code with PaCoRe-low inference setting: +```bash +python playground/example_batch_inference_pacore_low_1210.py +``` + +## 🙏 Acknowledgements +- This work was supported by computing resources and infrastructure provided by [StepFun](https://www.stepfun.com/) and Tsinghua University. +- We are built on amazing open source models and data; thanks again! + +## 📜 Citation + +```bibtex +@misc{pacore2025, + title={PaCoRe: Learning to Scale Test-Time Compute with Parallel Coordinated Reasoning}, + author={Jingcheng Hu and Yinmin Zhang and Shijie Shang and Xiaobo Yang and Yue Peng and Zhewei Huang and Hebin Zhou and Xin Wu and Jie Cheng and Fanqi Wan and Xiangwen Kong and Chengyuan Yao and Kaiwen Yan and Ailin Huang and Hongyu Zhou and Qi Han and Zheng Ge and Daxin Jiang and Xiangyu Zhang and Heung-Yeung Shum}, + year={2026}, + eprint={2601.05593}, + archivePrefix={arXiv}, + primaryClass={cs.LG}, + url={https://arxiv.org/abs/2601.05593}, +} +``` \ No newline at end of file diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..b54f913 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..6be1ad4 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,51 @@ +{% macro render_content(content) %}{% if content is none %}{{- "" }}{% elif content is string %}{{- content }}{% elif content is mapping %}{{- content['value'] if 'value' in content else content['text'] }}{% elif content is iterable %}{% for item in content %}{% if item.type == 'text' %}{{- item['value'] if 'value' in item else item['text'] }}{% elif item.type == 'image' %}{% endif %}{% endfor %}{% endif %}{% endmacro %}{%- if tools %}{{- '<|im_start|>system +' }}{%- if messages[0]['role'] == 'system' %}{{- render_content(messages[0]['content']) }}{%- else %}{{- '' }}{%- endif %}{{- " + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: +" }}{%- for tool in tools %}{{- " +" }}{{- tool | tojson }}{%- endfor %}{{- " + + +For each function call, return a json object with function name and arguments within XML tags: + +{\"name\": , \"arguments\": } +<|im_end|> +" }}{%- else %}{%- if messages[0]['role'] == 'system' %}{{- '<|im_start|>system +' + render_content(messages[0]['content']) + '<|im_end|> +' }}{%- endif %}{%- endif %}{%- for message in messages %}{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}{{- '<|im_start|>' + message.role + ' +' + render_content(message.content) + '<|im_end|>' + ' +' }}{%- elif message.role == "assistant" %}{%- if loop.last %}{{- '<|im_start|>assistant + +' + render_content(message.reasoning_content) + ' + +' + render_content(message.content) }}{%- if message.tool_calls %}{%- for tool_call in message.tool_calls %}{%- set call_details = tool_call.function if tool_call.function is defined else tool_call %}{%- set tool_call_id = tool_call.id if tool_call.id is defined else tool_call.tool_call_id %}{{- ' + +{"tool_call_id": "' + tool_call_id + '", "name": "' + call_details.name + '", "arguments": ' }}{% if call_details.arguments is string %}{{- call_details.arguments }}{% else %}{{- call_details.arguments | tojson }}{% endif %}{{- '} +' }}{%- endfor %}{%- endif %}{{- '<|im_end|> +' }}{%- else %}{{- '<|im_start|>assistant +' + render_content(message.content) }}{%- if message.tool_calls %}{%- for tool_call in message.tool_calls %}{%- set call_details = tool_call.function if tool_call.function is defined else tool_call %}{%- set tool_call_id = tool_call.id if tool_call.id is defined else tool_call.tool_call_id %}{{- ' + +{"tool_call_id": "' + tool_call_id + '", "name": "' + call_details.name + '", "arguments": ' }}{% if call_details.arguments is string %}{{- call_details.arguments }}{% else %}{{- call_details.arguments | tojson }}{% endif %}{{- '} +' }}{%- endfor %}{%- endif %}{{- '<|im_end|> +' }}{%- endif %} +{%- elif message.role in ["tool_response", "tool"] %} + {%- if loop.first or loop.previtem.role not in ["tool", "tool_response"] -%} + {{- '<|im_start|>tool_response +' -}} + {%- endif -%} + {{- ' +' + 'tool_call_id: ' + message.tool_call_id + ' +' + render_content(message.content) + ' + +' -}} + {%- if loop.last or loop.nextitem.role not in ["tool", "tool_response"] -%} + {{- '<|im_end|> +' -}} + {%- endif -%} +{%- endif %}{%- endfor %}{%- if add_generation_prompt %}{{- '<|im_start|>assistant + +' }}{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..e9b433c --- /dev/null +++ b/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "max_position_embeddings": 327680, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "factor": 2.0, + "low_freq_factor": 1.0, + "high_freq_factor": 32.0, + "original_max_position_embeddings": 65536, + "rope_type": "llama3" + }, + "rope_theta": 1000000, + "sliding_window": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} \ No newline at end of file diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..159097f --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "others", "allow_remote": true} \ No newline at end of file diff --git a/figure/before_after_train_lcb_02.png b/figure/before_after_train_lcb_02.png new file mode 100644 index 0000000..31235f2 Binary files /dev/null and b/figure/before_after_train_lcb_02.png differ diff --git a/figure/benchmark_accuracy_1130.png b/figure/benchmark_accuracy_1130.png new file mode 100644 index 0000000..431cbea Binary files /dev/null and b/figure/benchmark_accuracy_1130.png differ diff --git a/figure/inference_pipeline_teaser_02.png b/figure/inference_pipeline_teaser_02.png new file mode 100644 index 0000000..8d76e25 Binary files /dev/null and b/figure/inference_pipeline_teaser_02.png differ diff --git a/figure/teaser_draft_02.png b/figure/teaser_draft_02.png new file mode 100644 index 0000000..3a06c37 Binary files /dev/null and b/figure/teaser_draft_02.png differ diff --git a/figure/train_reward_response_length_1130.png b/figure/train_reward_response_length_1130.png new file mode 100644 index 0000000..848cc1e --- /dev/null +++ b/figure/train_reward_response_length_1130.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eb1900913eb5accaf1d2d3ef8c86799b2627aab6b5e5427e262b734a7cf17d1 +size 1067621 diff --git a/merges.txt b/merges.txt new file mode 100644 index 0000000..80c1a19 --- /dev/null +++ b/merges.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5 +size 1671853 diff --git a/model-00001.safetensors b/model-00001.safetensors new file mode 100644 index 0000000..da01761 --- /dev/null +++ b/model-00001.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d66e2175e7cc24877ca5c9333724af4ea3d508b876eb2d26f941847aa0c4f414 +size 3087152968 diff --git a/model-00002.safetensors b/model-00002.safetensors new file mode 100644 index 0000000..4e9efa9 --- /dev/null +++ b/model-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c3c4dc9cf1d410c4d323ee8c0ed59480a9bdfe820fb79e815eef73ee194fdb0 +size 4231132880 diff --git a/model-00003.safetensors b/model-00003.safetensors new file mode 100644 index 0000000..4b2df77 --- /dev/null +++ b/model-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d34a7e57f4d263661180ae83307091e141a31c4329e9f5e5a062d0eccc015c2 +size 4244835472 diff --git a/model-00004.safetensors b/model-00004.safetensors new file mode 100644 index 0000000..f03537c --- /dev/null +++ b/model-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec1435daa2550e910345b503b16b9183c4ba66cf9b36f992232605b71fb7b262 +size 3573735504 diff --git a/model-00005.safetensors b/model-00005.safetensors new file mode 100644 index 0000000..d29f776 --- /dev/null +++ b/model-00005.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c067161eaa1360e668b3fff1e901f1b4c466c2552eab309116c1e483f6d4a297 +size 1244659808 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..359e737 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,404 @@ +{ + "metadata": {}, + "weight_map": { + "model.layers.0.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model-00001.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model-00001.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model-00001.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model-00001.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model-00001.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model-00001.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model-00001.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model-00001.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model-00001.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model-00001.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model-00001.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model-00001.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model-00001.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model-00001.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model-00001.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model-00001.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001.safetensors", + "model.embed_tokens.weight": "model-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model-00002.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model-00002.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model-00002.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model-00002.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model-00002.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model-00002.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model-00002.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model-00002.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model-00003.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model-00003.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model-00003.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model-00003.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model-00003.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model-00003.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model-00003.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model-00003.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model-00003.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model-00003.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model-00003.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model-00003.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00004.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model-00004.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00004.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model-00004.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00004.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model-00004.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00004.safetensors", + "model.layers.30.self_attn.q_norm.weight": "model-00004.safetensors", + "model.layers.30.self_attn.k_norm.weight": "model-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00004.safetensors", + "model.layers.31.self_attn.q_norm.weight": "model-00004.safetensors", + "model.layers.31.self_attn.k_norm.weight": "model-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00004.safetensors", + "model.layers.32.self_attn.q_norm.weight": "model-00004.safetensors", + "model.layers.32.self_attn.k_norm.weight": "model-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004.safetensors", + "model.layers.33.self_attn.q_norm.weight": "model-00004.safetensors", + "model.layers.33.self_attn.k_norm.weight": "model-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004.safetensors", + "model.layers.34.self_attn.q_norm.weight": "model-00004.safetensors", + "model.layers.34.self_attn.k_norm.weight": "model-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004.safetensors", + "model.layers.35.self_attn.q_norm.weight": "model-00004.safetensors", + "model.layers.35.self_attn.k_norm.weight": "model-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004.safetensors", + "model.norm.weight": "model-00004.safetensors", + "lm_head.weight": "model-00005.safetensors" + } +} \ No newline at end of file diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..ac23c0a --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..cd71f61 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4 +size 11422654 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ddaf698 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,239 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/vocab.json b/vocab.json new file mode 100644 index 0000000..6c49fc6 --- /dev/null +++ b/vocab.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910 +size 2776833