初始化项目，由ModelHub XC社区提供模型
Model: meituan-longcat/UNO-Scorer-Qwen3-14B Source: Original Platform
This commit is contained in:
ModelHub XC
57
.gitattributes
vendored
Normal file
57
.gitattributes
vendored
Normal file
@@ -0,0 +1,57 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||
*.tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
*.db* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ark* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.gguf* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ggml filter=lfs diff=lfs merge=lfs -text
|
||||
*.llamafile* filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
model-00004-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
model-00005-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
model-00002-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
model-00006-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
merges.txt filter=lfs diff=lfs merge=lfs -text
|
||||
vocab.json filter=lfs diff=lfs merge=lfs -text
|
||||
model-00003-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
model-00001-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
202
LICENSE
Normal file
202
LICENSE
Normal file
@@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2024 Alibaba Cloud
|
||||
Copyright 2025 Meituan
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
320
README.md
Normal file
320
README.md
Normal file
@@ -0,0 +1,320 @@
|
||||
---
|
||||
language:
|
||||
- zh
|
||||
- en
|
||||
license: apache-2.0
|
||||
base_model: Qwen/Qwen3-14B
|
||||
library_name: transformers
|
||||
tags:
|
||||
- qwen
|
||||
- scoring
|
||||
- grading
|
||||
- evaluation
|
||||
- llm-judge
|
||||
pipeline_tag: text-generation
|
||||
---
|
||||
|
||||
# UNO-Scorer: A Unified General Scoring Model for UNO-Bench
|
||||
|
||||
<div align="center">
|
||||
|
||||
[](https://meituan-longcat.github.io/UNO-Bench)
|
||||
[](https://arxiv.org/abs/2510.18915)
|
||||
[](https://huggingface.co/Qwen/Qwen3-14B)
|
||||
[]()
|
||||
|
||||
</div>
|
||||
|
||||
## 📖 Introduction
|
||||
|
||||
**UNO-Scorer** is a lightweight yet high-precision **LLM-based evaluation model** designed to efficiently automate the evaluation of Large Multimodal Models (LMMs) with minimal computational overhead.
|
||||
|
||||
**Core Functionality:**
|
||||
- **Input**: Question + Reference Answer + Model Response
|
||||
- **Processing**: Analyzes correctness by **comparing each sub-question** against the reference answer
|
||||
- **Output**: Numerical score + Detailed evaluation reasoning for each sub-question
|
||||
|
||||
Built upon the powerful **Qwen3-14B** backbone, UNO-Scorer is fine-tuned on 13K high-quality in-house data. It overcomes the limitations of traditional Overall Reward Models (ORMs) by supporting **6 distinct question types**, with particular excellence in **Multi-Step Open-Ended Questions (MO)**.
|
||||
|
||||
|
||||
## 📊 Performance
|
||||
|
||||
UNO-Scorer demonstrates superior performance in automated evaluation, particularly in handling complex **Multi-Step Open-Ended Questions**. We compared the accuracy of our scorer against other advanced evaluators on our test set:
|
||||
|
||||
| Model | Accuracy |
|
||||
| :--- | :--- |
|
||||
| Seed-1.5-VL | 0.9118 |
|
||||
| GPT-4.1 | 0.9457 |
|
||||
| **UNO-Scorer (Ours)** | **0.9505** |
|
||||
|
||||
Experiments show that UNO-Scorer surpasses even proprietary frontier models like GPT-4.1 in this specific evaluation domain with lower cost.
|
||||
|
||||
## 💻 Usage
|
||||
|
||||
### ⚡ Quick Start (HuggingFace Transformers)
|
||||
|
||||
Get started with UNO-Scorer in just a few lines of code:
|
||||
|
||||
```bash
|
||||
pip install -U transformers
|
||||
python3 test_scorer_hf.py --model-name /path/to/UNO-Scorer
|
||||
```
|
||||
|
||||
**Minimal Example:**
|
||||
> ⚠️ **Critical**: The prompt template below is simplified for illustration. **Only the complete prompt template in `test_scorer_hf.py` will properly activate the model's fine-tuned scoring capabilities.** Custom or simplified prompts will not achieve optimal results.
|
||||
<details>
|
||||
<summary><b>Click to expand complete prompt template</b></summary>
|
||||
|
||||
```python
|
||||
def process_score_prompt(question, reference, response):
|
||||
promt_template = """请先通读问题信息，然后基于参考答案对模型回复的结果进行正确性打分。每道题可能包含多个小问，每个小问都已给出了相应的参考答案和分值，请逐小问校验模型回复是否正确，正确得对应分值，错误或漏答得0分，累计计分，有如下要求。
|
||||
|
||||
---
|
||||
|
||||
### 要求1：信息梳理
|
||||
|
||||
- 梳理出如下信息
|
||||
- 问题内容
|
||||
- 参考答案（可适度完善表达，但不改变核心内容）
|
||||
- 模型回复（需要将模型回复中的指代关系与参考答案对齐）
|
||||
- 分值
|
||||
|
||||
### 要求2：判断题型
|
||||
|
||||
- 明确该小问属于以下哪种题型之一，并基于该类型的打分标准进行打分，需要给出详细的比对过程。
|
||||
- **数值型**，要求模型回复与标准答案的数值完全相同，不允许有误差。例，`问题：北京奥运会是哪一年？参考答案：2008，模型回复：2004，打分结果：错误。`
|
||||
- **枚举型**，要求模型回复列举出参考答案的全部对象，缺一不可、错一不可，允许同义词等语义相近的表达，题中有顺序要求则必须按顺序枚举。例，`图中出现了哪些动物？参考答案：大熊猫、河马、长颈鹿，模型回复：河马、小熊猫、长颈鹿，打分结果：错误。 `注：“/”表示“或”，如，XXA/XXB，表示回答出任意一项即可。
|
||||
- **选择题**，要求模型回复与参考答案相同的选项或选项内容。例，`问题：李白是哪个朝代的诗人？A. 唐朝 B. 宋朝 C. 元朝，模型回复：李白是唐朝诗人，打分结果：正确。`
|
||||
- **判断题**，要求模型回复与参考答案的判断一致。例，`问题：图中鼠标是否放在了笔记本电脑左侧？参考答案：是，模型回复：图中鼠标在笔记本电脑的左侧。打分结果：正确。`
|
||||
- **简答题**，要求模型回复包括与参考答案语义一致的短语或表达，允许表达方式不同。例，`问题：视频中最后放入锅中的食材是什么？参考答案：洋葱，模型回复：胡萝卜。打分结果：错误。`
|
||||
- **论述题**，要求模型回复包含参考答案的核心观点。例，`问题：请简要论述为什么要保护生物多样性。参考答案：维持生态平衡，模型回复：保护生物多样性能够让生态系统保持稳定，促进人类社会的可持续发展。打分结果：正确。`
|
||||
|
||||
### 要求3：打分标准
|
||||
|
||||
- **完全正确**：得满分。
|
||||
- **错误或漏答**：得0分。
|
||||
- 如模型回复与参考答案大意相同但细节略有差别，且非核心内容，视为正确，具体参考参考答案的详细要求。
|
||||
- 若模型回复未直接给出答案，需主动归纳总结结论，只关注结论是否一致。
|
||||
- 每小问独立打分，前序错误不影响后续小问的结果。
|
||||
|
||||
### 要求4：输出格式
|
||||
|
||||
- 逐小问列出得分说明。
|
||||
- 所有小问得分相加，在<score></score>中给出总分，例如：<score>5</score>
|
||||
|
||||
---
|
||||
|
||||
## 问题信息
|
||||
{{question}}
|
||||
## 参考答案
|
||||
{{reference}}
|
||||
## 模型回复
|
||||
{{response}}
|
||||
## 逐小问打分"""
|
||||
|
||||
prompt = promt_template.replace("{{question}}", remove_thought_block(question.strip()))
|
||||
prompt = prompt.replace("{{reference}}", reference)
|
||||
prompt = prompt.replace("{{response}}", response)
|
||||
return prompt
|
||||
```
|
||||
</details>
|
||||
|
||||
|
||||
```python
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
import re
|
||||
|
||||
def extract_score(text):
|
||||
matches = re.findall(r'<score>([\d.]+)</score>', text)
|
||||
return float(matches[-1]) if matches else 0.0
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("meituan-longcat/UNO-Scorer-Qwen3-14B")
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
"meituan-longcat/UNO-Scorer-Qwen3-14B",
|
||||
torch_dtype="auto",
|
||||
device_map="auto"
|
||||
)
|
||||
|
||||
# Prepare scoring prompt
|
||||
question = "Which animal appears in the image?"
|
||||
reference = "Sub-question 1: Elephant, total score 10 points"
|
||||
response = "I see an elephant in the image."
|
||||
|
||||
# This prompt template is simplified for illustration.
|
||||
prompt = f"""Please score the model's response based on the reference answer.
|
||||
|
||||
Question: {question}
|
||||
Reference Answer: {reference}
|
||||
Model Response: {response}
|
||||
|
||||
Provide a step-by-step analysis and output the total score in <score></score> tags."""
|
||||
|
||||
# Generate score
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
||||
model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
|
||||
generated_ids = model.generate(
|
||||
**model_inputs,
|
||||
max_new_tokens=4096,
|
||||
do_sample=False
|
||||
)
|
||||
output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist()
|
||||
result = tokenizer.decode(output_ids, skip_special_tokens=True)
|
||||
print("Score response:\n", result)
|
||||
score = extract_score(result)
|
||||
print(f"Score: {score}")
|
||||
```
|
||||
|
||||
### 🔄 How It Works
|
||||
|
||||
UNO-Scorer evaluates model responses through a structured process:
|
||||
|
||||
1. **Information Organization**: Extracts question content, reference answer, model response, and scoring criteria
|
||||
2. **Question Type Classification**: Identifies the question type (multiple-choice, numerical, enumeration, yes/no, short-answer, or essay)
|
||||
3. **Detailed Comparison**: Compares model response against reference answer using type-specific criteria
|
||||
4. **Score Extraction**: Outputs final score in `<score>X</score>` format (where X is 0-10)
|
||||
|
||||
### 📥 Input Format Requirements
|
||||
|
||||
The model expects three key inputs:
|
||||
|
||||
| Component | Description | Example |
|
||||
| :--- | :--- | :--- |
|
||||
| **Question** | The original question posed to the model | "Which animals appear in the image?" |
|
||||
| **Reference Answer** | Ground truth answer with point allocation (sum to 10) | `Sub-question 1: Elephant, total score 10 points` |
|
||||
| **Model Response** | The response from the model being evaluated | "I see an elephant in the image." |
|
||||
|
||||
#### Reference Answer Formatting (Critical!)
|
||||
|
||||
Since the model is trained primarily on Chinese corpora, **formatting reference answers in Chinese yields significantly better results**. However, English formatting is also supported.
|
||||
|
||||
**For Single-Answer Questions:**
|
||||
```
|
||||
1. {Answer}, total score 10 points, focus only on final answer correctness
|
||||
1. {答案}，总分10分，无需关注推理过程，最终答案正确即可
|
||||
```
|
||||
**For Multi-Part Questions:**
|
||||
```
|
||||
1. {Sub-Answer A} ({X} points); 2. {Sub-Answer B} ({Y} points)
|
||||
1. {子答案A}（{X}分）; 2. {子答案B}（{Y}分）
|
||||
```
|
||||
|
||||
**With Custom Scoring Criteria:**
|
||||
|
||||
```
|
||||
1. {Answer}, total score 10 points, scoring criteria: {detailed criteria}
|
||||
1. {答案}，总分10分，评分标准：{详细标准}
|
||||
```
|
||||
|
||||
### 📤 Output Format
|
||||
|
||||
The model returns:
|
||||
|
||||
- **Detailed Evaluation**: Step-by-step analysis for each sub-question
|
||||
- **Score Tag**: `<score>X</score>` where X ranges from 0 to 10
|
||||
|
||||
Example output:
|
||||
```
|
||||
Sub-question 1:
|
||||
|
||||
Question Content: How many apples are in the image?
|
||||
Reference Answer: 2
|
||||
Model Response: There are two appels.
|
||||
Points: 10 point
|
||||
Question Type: Numerical
|
||||
Comparison Process: The reference answer is "2" and the model response is "two". The numerical values are completely identical, with only the expression format differing. This meets the scoring standard for numerical questions.
|
||||
|
||||
Scoring Explanation: Completely correct, awarded 10 point.
|
||||
|
||||
<score>10</score>
|
||||
```
|
||||
|
||||
### 📋 Complete Evaluation Example
|
||||
|
||||
See `test_scorer_hf.py` for a full working example with multiple question types:
|
||||
|
||||
- Multiple-choice questions
|
||||
- Yes/No questions
|
||||
- Open-ended questions
|
||||
- Multi-part questions
|
||||
|
||||
Run the example:
|
||||
```bash
|
||||
python3 test_scorer_hf.py --model-name /path/to/UNO-Scorer
|
||||
```
|
||||
|
||||
### 🚀 Optimized Inference with vLLM (Recommended for Production)
|
||||
|
||||
For large-scale evaluation tasks, we strongly recommend using **vLLM** for significant performance improvements:
|
||||
|
||||
```bash
|
||||
# 1. Clone the repository
|
||||
git clone https://github.com/meituan-longcat/UNO-Bench.git
|
||||
cd UNO-Bench/uno-eval
|
||||
|
||||
# 2. Install dependencies
|
||||
pip install -r requirements.txt
|
||||
|
||||
# 3. Run vLLM-based inference
|
||||
bash examples/test_scorer_vllm.sh
|
||||
```
|
||||
|
||||
**Why vLLM?**
|
||||
|
||||
- **10-20x faster** inference compared to standard HuggingFace
|
||||
- Better batching support for multiple evaluation tasks
|
||||
- Lower memory footprint
|
||||
- Optimized for production deployments
|
||||
|
||||
### ⚠️ Important Notes
|
||||
|
||||
1. **Language**: Chinese formatting in reference answers produces significantly better results due to the model's training data composition
|
||||
2. **Point Allocation**: Reference answers must have total points equal to 10
|
||||
3. **Score Extraction**: Always look for `<score>X</score>` in the output
|
||||
4. **Batch Processing**: Use vLLM for evaluating multiple responses efficiently
|
||||
5. **Question Type Awareness**: Ensure reference answers clearly specify the question type for optimal scoring
|
||||
|
||||
## 🎯 Supported Question Types
|
||||
|
||||
UNO-Scorer supports evaluation across 6 distinct question types:
|
||||
|
||||
| Question Type | Description | Scoring Rule |
|
||||
| :--- | :--- | :--- |
|
||||
| **Multiple-Choice** | Select correct option from given choices | Response must match the correct option exactly |
|
||||
| **Numerical** | Provide specific numerical values | No tolerance for numerical errors |
|
||||
| **Enumeration** | List all required items | Must include all items, no omissions or errors |
|
||||
| **Yes/No** | Binary judgment questions | Response judgment must match reference answer |
|
||||
| **Short-Answer** | Brief factual answers | Semantic equivalence acceptable, expression flexibility allowed |
|
||||
| **Essay** | Longer analytical responses | Must contain core viewpoints from reference answer |
|
||||
|
||||
## 📜 Citation
|
||||
|
||||
If you find this model or the UNO-Bench useful for your research, please cite our paper:
|
||||
|
||||
```bibtex
|
||||
@misc{chen2025unobench,
|
||||
title={UNO-Bench: A Unified Benchmark for Exploring the Compositional Law Between Uni-modal and Omni-modal in Omni Models},
|
||||
author={Chen Chen and ZeYang Hu and Fengjiao Chen and Liya Ma and Jiaxing Liu and Xiaoyu Li and Ziwen Wang and Xuezhi Cao and Xunliang Cai},
|
||||
year={2025},
|
||||
eprint={2510.18915},
|
||||
archivePrefix={arXiv},
|
||||
primaryClass={cs.CL},
|
||||
url={https://arxiv.org/abs/2510.18915},
|
||||
}
|
||||
```
|
||||
|
||||
## ⚖️ License & Disclaimer
|
||||
|
||||
This model is released under the **Apache 2.0 License**. It is based on Qwen3-14B. Please strictly follow the license and usage policy of the original Qwen model series.
|
||||
|
||||
**Disclaimer**: This model is designed for research and evaluation purposes. Users are responsible for ensuring their use complies with applicable laws and regulations.
|
||||
|
||||
## 🤝 Contributing
|
||||
|
||||
We welcome contributions and feedback! Please feel free to:
|
||||
- Report issues or bugs
|
||||
- Suggest improvements
|
||||
- Share your evaluation results
|
||||
- Contribute enhancements
|
||||
|
||||
For more information, visit our [GitHub repository](https://github.com/meituan-longcat/UNO-Bench).
|
||||
28
added_tokens.json
Normal file
28
added_tokens.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"</think>": 151668,
|
||||
"</tool_call>": 151658,
|
||||
"</tool_response>": 151666,
|
||||
"<think>": 151667,
|
||||
"<tool_call>": 151657,
|
||||
"<tool_response>": 151665,
|
||||
"<|box_end|>": 151649,
|
||||
"<|box_start|>": 151648,
|
||||
"<|endoftext|>": 151643,
|
||||
"<|file_sep|>": 151664,
|
||||
"<|fim_middle|>": 151660,
|
||||
"<|fim_pad|>": 151662,
|
||||
"<|fim_prefix|>": 151659,
|
||||
"<|fim_suffix|>": 151661,
|
||||
"<|im_end|>": 151645,
|
||||
"<|im_start|>": 151644,
|
||||
"<|image_pad|>": 151655,
|
||||
"<|object_ref_end|>": 151647,
|
||||
"<|object_ref_start|>": 151646,
|
||||
"<|quad_end|>": 151651,
|
||||
"<|quad_start|>": 151650,
|
||||
"<|repo_name|>": 151663,
|
||||
"<|video_pad|>": 151656,
|
||||
"<|vision_end|>": 151653,
|
||||
"<|vision_pad|>": 151654,
|
||||
"<|vision_start|>": 151652
|
||||
}
|
||||
30
config.json
Normal file
30
config.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen3ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 151643,
|
||||
"eos_token_id": 151645,
|
||||
"head_dim": 128,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 5120,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 17408,
|
||||
"max_position_embeddings": 40960,
|
||||
"max_window_layers": 40,
|
||||
"model_type": "qwen3",
|
||||
"num_attention_heads": 40,
|
||||
"num_hidden_layers": 40,
|
||||
"num_key_value_heads": 8,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_scaling": null,
|
||||
"rope_theta": 1000000,
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.51.0",
|
||||
"use_cache": false,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
1
configuration.json
Normal file
1
configuration.json
Normal file
@@ -0,0 +1 @@
|
||||
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}
|
||||
13
generation_config.json
Normal file
13
generation_config.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"bos_token_id": 151643,
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"temperature": 0.6,
|
||||
"top_k": 20,
|
||||
"top_p": 0.95,
|
||||
"transformers_version": "4.51.0"
|
||||
}
|
||||
BIN
merges.txt
(Stored with Git LFS)
Normal file
BIN
merges.txt
(Stored with Git LFS)
Normal file
Binary file not shown.
3
model-00001-of-00006.safetensors
Normal file
3
model-00001-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:bdb25da8e44943b0d0c4ae36ef642823a85cd73e88d837d3741ef0ada03af74f
|
||||
size 4984780784
|
||||
3
model-00002-of-00006.safetensors
Normal file
3
model-00002-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5f10f2ce02ecbff623315b99527ee701752f22de57d8b17402b2ec7eec5e92bb
|
||||
size 4980892048
|
||||
3
model-00003-of-00006.safetensors
Normal file
3
model-00003-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a82a8b6e9fd7bca4e5a160fed235c9d0c1a51e1b130e25144291d3fcc67971de
|
||||
size 4928485104
|
||||
3
model-00004-of-00006.safetensors
Normal file
3
model-00004-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1c3d62883a9ba5cae605192a27066ce27d6b1dca4f30a9aae1ba3ed02f9e8482
|
||||
size 4980892112
|
||||
3
model-00005-of-00006.safetensors
Normal file
3
model-00005-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:85fb9f1a718b34a7a08bd56e5862f27fc7e4956128a03ece1b04dd2e1bd82f2e
|
||||
size 4928485104
|
||||
3
model-00006-of-00006.safetensors
Normal file
3
model-00006-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:df79aad074fc95e9706491ababe75f872d20e8b0afff2d1d6a56bdf955a3f9f6
|
||||
size 4733130504
|
||||
450
model.safetensors.index.json
Normal file
450
model.safetensors.index.json
Normal file
@@ -0,0 +1,450 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_size": 29536614400
|
||||
},
|
||||
"weight_map": {
|
||||
"lm_head.weight": "model-00006-of-00006.safetensors",
|
||||
"model.embed_tokens.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.10.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.12.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.13.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.18.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.18.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.18.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.18.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.19.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.19.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.19.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.19.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.2.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.20.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.k_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.q_norm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.21.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.24.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.24.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.24.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.24.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.25.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.25.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.25.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.25.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.26.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.26.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.26.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.26.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.27.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.27.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.k_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.q_norm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.28.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.3.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.30.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.31.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.31.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.31.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.31.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.31.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.32.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.32.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.32.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.32.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.32.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.32.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.32.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.32.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.32.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.32.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.32.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.33.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.33.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.33.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.33.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.33.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.33.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.33.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.33.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.33.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.33.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.33.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.34.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.34.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.34.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.34.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.34.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.34.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.34.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.34.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.34.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.34.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.34.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.35.input_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.35.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.35.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.35.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.35.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.35.self_attn.k_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.35.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.35.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.35.self_attn.q_norm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.35.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.35.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.36.input_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.36.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.36.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.36.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.36.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.36.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.36.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.36.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.36.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.36.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.36.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.37.input_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.37.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.37.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.37.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.37.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.37.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.37.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.37.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.37.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.37.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.37.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.38.input_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.38.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.38.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.38.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.38.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.38.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.38.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.38.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.38.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.38.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.38.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.39.input_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.39.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.39.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.39.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.39.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.39.self_attn.k_norm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.39.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.39.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.39.self_attn.q_norm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.39.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.39.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.4.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.5.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.k_norm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.q_norm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.norm.weight": "model-00006-of-00006.safetensors"
|
||||
}
|
||||
}
|
||||
31
special_tokens_map.json
Normal file
31
special_tokens_map.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
194
test_scorer_hf.py
Normal file
194
test_scorer_hf.py
Normal file
@@ -0,0 +1,194 @@
|
||||
import re
|
||||
import argparse
|
||||
from tqdm import tqdm
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
def extract_last_boxed(text):
|
||||
try:
|
||||
pattern = r'<score>([\d.]+)</score>'
|
||||
matches = re.findall(pattern, text)
|
||||
if matches:
|
||||
return float(matches[-1])
|
||||
else:
|
||||
return 0.0
|
||||
except Exception as e:
|
||||
print(f"Error extracting boxed content: {e}")
|
||||
return 0.0
|
||||
|
||||
def parse_from_score_model(response: str, scale_factor=10) -> float:
|
||||
score = extract_last_boxed(response)
|
||||
score = score / scale_factor
|
||||
return score
|
||||
|
||||
def load_model(model_name: str) -> AutoModelForCausalLM:
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_name,
|
||||
torch_dtype="auto",
|
||||
device_map="auto"
|
||||
)
|
||||
return tokenizer, model
|
||||
|
||||
def generate(model, tokenizer, prompt: str) -> str:
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful assistant."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": prompt
|
||||
}
|
||||
]
|
||||
text = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
|
||||
|
||||
# conduct text completion
|
||||
generated_ids = model.generate(
|
||||
**model_inputs,
|
||||
max_new_tokens=16384,
|
||||
do_sample=False
|
||||
)
|
||||
output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist()
|
||||
|
||||
content = tokenizer.decode(output_ids, skip_special_tokens=True)
|
||||
return content
|
||||
|
||||
def remove_thought_block(text: str) -> str:
|
||||
pattern = r"^(<think>.*?</think>|.*?)"
|
||||
match = re.match(pattern, text, flags=re.DOTALL)
|
||||
if match:
|
||||
end_of_match = match.end()
|
||||
return text[end_of_match:].lstrip()
|
||||
return text
|
||||
|
||||
def process_score_prompt(question, reference, response):
|
||||
promt_template = """请先通读问题信息，然后基于参考答案对模型回复的结果进行正确性打分。每道题可能包含多个小问，每个小问都已给出了相应的参考答案和分值，请逐小问校验模型回复是否正确，正确得对应分值，错误或漏答得0分，累计计分，有如下要求。
|
||||
|
||||
---
|
||||
|
||||
### 要求1：信息梳理
|
||||
|
||||
- 梳理出如下信息
|
||||
- 问题内容
|
||||
- 参考答案（可适度完善表达，但不改变核心内容）
|
||||
- 模型回复（需要将模型回复中的指代关系与参考答案对齐）
|
||||
- 分值
|
||||
|
||||
### 要求2：判断题型
|
||||
|
||||
- 明确该小问属于以下哪种题型之一，并基于该类型的打分标准进行打分，需要给出详细的比对过程。
|
||||
- **数值型**，要求模型回复与标准答案的数值完全相同，不允许有误差。例，`问题：北京奥运会是哪一年？参考答案：2008，模型回复：2004，打分结果：错误。`
|
||||
- **枚举型**，要求模型回复列举出参考答案的全部对象，缺一不可、错一不可，允许同义词等语义相近的表达，题中有顺序要求则必须按顺序枚举。例，`图中出现了哪些动物？参考答案：大熊猫、河马、长颈鹿，模型回复：河马、小熊猫、长颈鹿，打分结果：错误。 `注：“/”表示“或”，如，XXA/XXB，表示回答出任意一项即可。
|
||||
- **选择题**，要求模型回复与参考答案相同的选项或选项内容。例，`问题：李白是哪个朝代的诗人？A. 唐朝 B. 宋朝 C. 元朝，模型回复：李白是唐朝诗人，打分结果：正确。`
|
||||
- **判断题**，要求模型回复与参考答案的判断一致。例，`问题：图中鼠标是否放在了笔记本电脑左侧？参考答案：是，模型回复：图中鼠标在笔记本电脑的左侧。打分结果：正确。`
|
||||
- **简答题**，要求模型回复包括与参考答案语义一致的短语或表达，允许表达方式不同。例，`问题：视频中最后放入锅中的食材是什么？参考答案：洋葱，模型回复：胡萝卜。打分结果：错误。`
|
||||
- **论述题**，要求模型回复包含参考答案的核心观点。例，`问题：请简要论述为什么要保护生物多样性。参考答案：维持生态平衡，模型回复：保护生物多样性能够让生态系统保持稳定，促进人类社会的可持续发展。打分结果：正确。`
|
||||
|
||||
### 要求3：打分标准
|
||||
|
||||
- **完全正确**：得满分。
|
||||
- **错误或漏答**：得0分。
|
||||
- 如模型回复与参考答案大意相同但细节略有差别，且非核心内容，视为正确，具体参考参考答案的详细要求。
|
||||
- 若模型回复未直接给出答案，需主动归纳总结结论，只关注结论是否一致。
|
||||
- 每小问独立打分，前序错误不影响后续小问的结果。
|
||||
|
||||
### 要求4：输出格式
|
||||
|
||||
- 逐小问列出得分说明。
|
||||
- 所有小问得分相加，在<score></score>中给出总分，例如：<score>5</score>
|
||||
|
||||
---
|
||||
|
||||
## 问题信息
|
||||
{{question}}
|
||||
## 参考答案
|
||||
{{reference}}
|
||||
## 模型回复
|
||||
{{response}}
|
||||
## 逐小问打分"""
|
||||
|
||||
prompt = promt_template.replace("{{question}}", remove_thought_block(question.strip()))
|
||||
prompt = prompt.replace("{{reference}}", reference)
|
||||
prompt = prompt.replace("{{response}}", response)
|
||||
return prompt
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
parser = argparse.ArgumentParser(description="Run evaluation on a given model and dataset.")
|
||||
parser.add_argument("--model-name", type=str, default="", help="The model name or path.")
|
||||
|
||||
args = parser.parse_args()
|
||||
tokenizer, model = load_model(args.model_name)
|
||||
|
||||
test_cases = [
|
||||
# Multiple-choice question
|
||||
{
|
||||
"question_type": "multiple-choice",
|
||||
"question": "Which one is the best depiction of this image?\nA. A beautiful woman wearing a hat is holding a rose in her right hand and is opening her mouth big to eat it.\nB. A young woman is wearing a hat and her right hand is pierced by a bouquet of pink roses, and she opens her mouth to express pain.\nC. A young girl wearing brown clothes is holding a blooming pink flower in her right hand, with her mouth slightly open.\nD. A young girl wearing brown clothes is holding a blooming magic wand in her right hand, with her mouth slightly open.",
|
||||
"answer": "C",
|
||||
"response": "Well, if we're talking about the best depiction of this image, I'd say it's C. The young girl in brown clothes holding a blooming pink flower in her right hand with her mouth slightly open seems to match the image the most. What do you think? Do you agree with that?",
|
||||
"target_score": 1
|
||||
},
|
||||
# Yes/No question
|
||||
{
|
||||
"question_type": "yes/no",
|
||||
"question": "The following images show two graphs. Each graph is represented in a separate image. Are these graphs isomorphic?Answer Yes or No.",
|
||||
"answer": "Yes",
|
||||
"response": "No. If you have any other questions about these graphs or anything else, feel free to ask!",
|
||||
"target_score": 0
|
||||
},
|
||||
# Open-ended question
|
||||
{
|
||||
"question_type": "open-ended",
|
||||
"question": "How many seasons did 90210 have?",
|
||||
"answer": "Ten",
|
||||
"response": "I'm not sure which \"90210\" you're referring to. There are different shows or things with that name. If it's the TV show \"90210\", it had 7 seasons. But if it's something else, you might need to give me a bit more context.",
|
||||
"target_score": 0
|
||||
},
|
||||
# Multiple-question answer
|
||||
{
|
||||
# If the question type is multi-question, you must specify the answer in detail.
|
||||
"question_type": "multiple-question",
|
||||
"question": "David is playing an escape room game. In one of the levels, he needs to find the key to the room. The game rules state that he must locate the box containing the key among a row of identical-looking boxes, and correctly enter the box's password to obtain the key to escape. It is known that the box with the key meets the following conditions: it is not at either end; it is not in the very middle; the number of boxes on both its left and right sides is even. In addition, each box has a three-digit password, with each digit being one of the numbers from 0 to 9. The arrangement of the boxes and the password puzzle are hidden in image <image_1>. The image puzzle provides two decryption examples, and you need to use these examples to decode which digits the two question marks at the bottom of the image represent. Among all the numbers given in the provided examples and obtained after decoding, the smallest number is interference information and should be excluded. Partial clues about the box containing the key and its password are hidden in audio <audio_1>. Based on the above information, and combining the image and audio clues, answer:\n1. Which box (counting from left to right) contains the key?\n2. What is the correct password for the box containing the key?",
|
||||
"answer": "1. The 5th box from the left (4 points);\n2. The password is 569 (6 points).",
|
||||
"response": "Hmm… let me think. According to the hint in the audio, the key to decryption lies in the Roman numerals hidden in the words, and these numbers are all no greater than 10. From the image, there are six boxes. Based on the rules, the key is neither at the ends nor in the middle, so it should be in the 5th box from the left. As for the password, arranging in ascending order, it should be Victory = 2, give = 3. So the password for the box with the key is 23. If you have any other ideas or questions, feel free to let me know.",
|
||||
"target_score": 0.4
|
||||
}
|
||||
]
|
||||
|
||||
prompts = []
|
||||
for case in test_cases:
|
||||
answer = case["answer"]
|
||||
if case["question_type"] != "multiple-question":
|
||||
# The Chinese rule will be better because the scorer model is trained in Chinese.
|
||||
answer = f"小问1：{answer}，总分10分，无需关注推理过程，最终答案正确即可"
|
||||
question = case["question"]
|
||||
response = remove_thought_block(case["response"])
|
||||
prompt = process_score_prompt(question=question, reference=answer, response=response)
|
||||
prompts.append(prompt)
|
||||
|
||||
score_responses = []
|
||||
for prompt in tqdm(prompts):
|
||||
score_response = generate(model, tokenizer, prompt)
|
||||
score_responses.append(score_response)
|
||||
|
||||
pass_cnt = 0
|
||||
for score_response, case in zip(score_responses, test_cases):
|
||||
print("="*32)
|
||||
score = parse_from_score_model(score_response)
|
||||
for key,value in case.items():
|
||||
print(f"{key}: {value}")
|
||||
print("Score response:\n", score_response)
|
||||
print(f"Score: {score}, Target Score: {case['target_score']}")
|
||||
|
||||
if score == case["target_score"]:
|
||||
pass_cnt += 1
|
||||
print("*"*32)
|
||||
print(f"Pass: {pass_cnt}/{len(test_cases)}")
|
||||
|
||||
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
Binary file not shown.
241
tokenizer_config.json
Normal file
241
tokenizer_config.json
Normal file
@@ -0,0 +1,241 @@
|
||||
{
|
||||
"add_bos_token": false,
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"151643": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151644": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151645": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151646": {
|
||||
"content": "<|object_ref_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151647": {
|
||||
"content": "<|object_ref_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151648": {
|
||||
"content": "<|box_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151649": {
|
||||
"content": "<|box_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151650": {
|
||||
"content": "<|quad_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151651": {
|
||||
"content": "<|quad_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151652": {
|
||||
"content": "<|vision_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151653": {
|
||||
"content": "<|vision_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151654": {
|
||||
"content": "<|vision_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151655": {
|
||||
"content": "<|image_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151656": {
|
||||
"content": "<|video_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151657": {
|
||||
"content": "<tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151658": {
|
||||
"content": "</tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151659": {
|
||||
"content": "<|fim_prefix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151660": {
|
||||
"content": "<|fim_middle|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151661": {
|
||||
"content": "<|fim_suffix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151662": {
|
||||
"content": "<|fim_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151663": {
|
||||
"content": "<|repo_name|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151664": {
|
||||
"content": "<|file_sep|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151665": {
|
||||
"content": "<tool_response>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151666": {
|
||||
"content": "</tool_response>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151667": {
|
||||
"content": "<think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151668": {
|
||||
"content": "</think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"bos_token": null,
|
||||
"chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set content = message.content %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is defined and message.reasoning_content is not none %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in message.content %}\n {%- set content = message.content.split('</think>')[-1].lstrip('\\n') %}\n {%- set reasoning_content = message.content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content.strip('\\n') + '\\n</think>\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- endif %}\n{%- endif %}",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": {},
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"padding_side": "right",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
BIN
vocab.json
(Stored with Git LFS)
Normal file
BIN
vocab.json
(Stored with Git LFS)
Normal file
Binary file not shown.
Reference in New Issue
Block a user