初始化项目，由ModelHub XC社区提供模型
Model: chanceQZhang/zhihu-tech-zhichang-qwen-7b Source: Original Platform
This commit is contained in:
ModelHub XC
41
.gitattributes
vendored
Normal file
41
.gitattributes
vendored
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
checkpoint-302/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||||
|
checkpoint-536/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||||
|
merged/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||||
|
merged/zhihu_f16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||||
|
merged/zhihu_model_q5_km.gguf filter=lfs diff=lfs merge=lfs -text
|
||||||
|
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||||
122
README.md
Normal file
122
README.md
Normal file
@@ -0,0 +1,122 @@
|
|||||||
|
---
|
||||||
|
library_name: peft
|
||||||
|
license: gpl-3.0
|
||||||
|
base_model: Orion-zhen/Meissa-Qwen2.5-7B-Instruct
|
||||||
|
tags:
|
||||||
|
- axolotl
|
||||||
|
- base_model:adapter:Orion-zhen/Meissa-Qwen2.5-7B-Instruct
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
datasets:
|
||||||
|
- chanceQZhang/zhihuhighvotes
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
model-index:
|
||||||
|
- name: outputs/zhihu-tech-career-lora
|
||||||
|
results: []
|
||||||
|
---
|
||||||
|
|
||||||
|
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
||||||
|
should probably proofread and complete it, then remove this comment. -->
|
||||||
|
|
||||||
|
[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
|
||||||
|
<details><summary>See axolotl config</summary>
|
||||||
|
|
||||||
|
axolotl version: `0.13.0.dev0`
|
||||||
|
```yaml
|
||||||
|
# config_sft_zhihu.yml
|
||||||
|
|
||||||
|
base_model: Orion-zhen/Meissa-Qwen2.5-7B-Instruct
|
||||||
|
model_type: AutoModelForCausalLM
|
||||||
|
tokenizer_type: AutoTokenizer
|
||||||
|
|
||||||
|
# 使用您上传的数据集
|
||||||
|
datasets:
|
||||||
|
- path: chanceQZhang/zhihuhighvotes
|
||||||
|
type: chat_template # ChatML 格式使用 chat_template
|
||||||
|
split: train
|
||||||
|
# 提速核心
|
||||||
|
sample_packing: true
|
||||||
|
pad_to_sequence_len: true
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# LoRA 配置
|
||||||
|
adapter: lora
|
||||||
|
lora_r: 8
|
||||||
|
lora_alpha: 32
|
||||||
|
lora_dropout: 0.1
|
||||||
|
lora_target_modules:
|
||||||
|
- q_proj
|
||||||
|
- v_proj
|
||||||
|
- k_proj
|
||||||
|
- o_proj
|
||||||
|
- gate_proj
|
||||||
|
- up_proj
|
||||||
|
- down_proj
|
||||||
|
|
||||||
|
# --- 核心优化：显存节省配置 ---
|
||||||
|
bf16: true # 30/40系列或A系列显卡必开，提升速度且省显存
|
||||||
|
fp16: false
|
||||||
|
gradient_checkpointing: true # 必开！用计算时间换空间，大幅降低显存占用
|
||||||
|
flash_attention: true # 必开！大幅降低长文本下的显存需求
|
||||||
|
|
||||||
|
# 训练配置
|
||||||
|
sequence_len: 2048
|
||||||
|
micro_batch_size: 6
|
||||||
|
gradient_accumulation_steps: 3
|
||||||
|
num_epochs: 2
|
||||||
|
learning_rate: 0.00005
|
||||||
|
# 减少中间开销
|
||||||
|
logging_steps: 10
|
||||||
|
eval_steps: 100
|
||||||
|
save_steps: 302
|
||||||
|
|
||||||
|
# 输出
|
||||||
|
output_dir: ./outputs/zhihu-tech-career-lora
|
||||||
|
```
|
||||||
|
|
||||||
|
</details><br>
|
||||||
|
|
||||||
|
# outputs/zhihu-tech-career-lora
|
||||||
|
|
||||||
|
This model is a fine-tuned version of [Orion-zhen/Meissa-Qwen2.5-7B-Instruct](https://huggingface.co/Orion-zhen/Meissa-Qwen2.5-7B-Instruct) on the chanceQZhang/zhihuhighvotes dataset.
|
||||||
|
|
||||||
|
## Model description
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Intended uses & limitations
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Training and evaluation data
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Training procedure
|
||||||
|
|
||||||
|
### Training hyperparameters
|
||||||
|
|
||||||
|
The following hyperparameters were used during training:
|
||||||
|
- learning_rate: 5e-05
|
||||||
|
- train_batch_size: 6
|
||||||
|
- eval_batch_size: 6
|
||||||
|
- seed: 42
|
||||||
|
- gradient_accumulation_steps: 3
|
||||||
|
- total_train_batch_size: 18
|
||||||
|
- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
|
||||||
|
- lr_scheduler_type: cosine
|
||||||
|
- lr_scheduler_warmup_steps: 16
|
||||||
|
- training_steps: 536
|
||||||
|
|
||||||
|
### Training results
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.18.1
|
||||||
|
- Transformers 4.57.1
|
||||||
|
- Pytorch 2.8.0+cu128
|
||||||
|
- Datasets 4.4.2
|
||||||
|
- Tokenizers 0.22.2
|
||||||
46
adapter_config.json
Normal file
46
adapter_config.json
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Orion-zhen/Meissa-Qwen2.5-7B-Instruct",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": null,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 32,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.1,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.18.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 8,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"o_proj",
|
||||||
|
"k_proj",
|
||||||
|
"q_proj",
|
||||||
|
"up_proj",
|
||||||
|
"down_proj",
|
||||||
|
"gate_proj",
|
||||||
|
"v_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": [],
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
adapter_model.safetensors
Normal file
3
adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f0380466a97b625606644758c2302e378fb50d91b23bdffa6dad1121c0439a23
|
||||||
|
size 80792096
|
||||||
24
added_tokens.json
Normal file
24
added_tokens.json
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
{
|
||||||
|
"</tool_call>": 151658,
|
||||||
|
"<tool_call>": 151657,
|
||||||
|
"<|box_end|>": 151649,
|
||||||
|
"<|box_start|>": 151648,
|
||||||
|
"<|endoftext|>": 151643,
|
||||||
|
"<|file_sep|>": 151664,
|
||||||
|
"<|fim_middle|>": 151660,
|
||||||
|
"<|fim_pad|>": 151662,
|
||||||
|
"<|fim_prefix|>": 151659,
|
||||||
|
"<|fim_suffix|>": 151661,
|
||||||
|
"<|im_end|>": 151645,
|
||||||
|
"<|im_start|>": 151644,
|
||||||
|
"<|image_pad|>": 151655,
|
||||||
|
"<|object_ref_end|>": 151647,
|
||||||
|
"<|object_ref_start|>": 151646,
|
||||||
|
"<|quad_end|>": 151651,
|
||||||
|
"<|quad_start|>": 151650,
|
||||||
|
"<|repo_name|>": 151663,
|
||||||
|
"<|video_pad|>": 151656,
|
||||||
|
"<|vision_end|>": 151653,
|
||||||
|
"<|vision_pad|>": 151654,
|
||||||
|
"<|vision_start|>": 151652
|
||||||
|
}
|
||||||
7
chat_template.jinja
Normal file
7
chat_template.jinja
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system
|
||||||
|
' + system_message + '<|im_end|>
|
||||||
|
' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user
|
||||||
|
' + content + '<|im_end|>
|
||||||
|
<|im_start|>assistant
|
||||||
|
' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '
|
||||||
|
' }}{% endif %}{% endfor %}
|
||||||
208
checkpoint-302/README.md
Normal file
208
checkpoint-302/README.md
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
---
|
||||||
|
base_model: Orion-zhen/Meissa-Qwen2.5-7B-Instruct
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- axolotl
|
||||||
|
- base_model:adapter:Orion-zhen/Meissa-Qwen2.5-7B-Instruct
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.18.1
|
||||||
46
checkpoint-302/adapter_config.json
Normal file
46
checkpoint-302/adapter_config.json
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Orion-zhen/Meissa-Qwen2.5-7B-Instruct",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": null,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 32,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.1,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.18.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 8,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"o_proj",
|
||||||
|
"k_proj",
|
||||||
|
"q_proj",
|
||||||
|
"up_proj",
|
||||||
|
"down_proj",
|
||||||
|
"gate_proj",
|
||||||
|
"v_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": [],
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-302/adapter_model.safetensors
Normal file
3
checkpoint-302/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:56bd0f8891bca69dda6c673dbc88ff6cf78f965711103973ca65bdaf4e6bd309
|
||||||
|
size 80792096
|
||||||
24
checkpoint-302/added_tokens.json
Normal file
24
checkpoint-302/added_tokens.json
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
{
|
||||||
|
"</tool_call>": 151658,
|
||||||
|
"<tool_call>": 151657,
|
||||||
|
"<|box_end|>": 151649,
|
||||||
|
"<|box_start|>": 151648,
|
||||||
|
"<|endoftext|>": 151643,
|
||||||
|
"<|file_sep|>": 151664,
|
||||||
|
"<|fim_middle|>": 151660,
|
||||||
|
"<|fim_pad|>": 151662,
|
||||||
|
"<|fim_prefix|>": 151659,
|
||||||
|
"<|fim_suffix|>": 151661,
|
||||||
|
"<|im_end|>": 151645,
|
||||||
|
"<|im_start|>": 151644,
|
||||||
|
"<|image_pad|>": 151655,
|
||||||
|
"<|object_ref_end|>": 151647,
|
||||||
|
"<|object_ref_start|>": 151646,
|
||||||
|
"<|quad_end|>": 151651,
|
||||||
|
"<|quad_start|>": 151650,
|
||||||
|
"<|repo_name|>": 151663,
|
||||||
|
"<|video_pad|>": 151656,
|
||||||
|
"<|vision_end|>": 151653,
|
||||||
|
"<|vision_pad|>": 151654,
|
||||||
|
"<|vision_start|>": 151652
|
||||||
|
}
|
||||||
7
checkpoint-302/chat_template.jinja
Normal file
7
checkpoint-302/chat_template.jinja
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system
|
||||||
|
' + system_message + '<|im_end|>
|
||||||
|
' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user
|
||||||
|
' + content + '<|im_end|>
|
||||||
|
<|im_start|>assistant
|
||||||
|
' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '
|
||||||
|
' }}{% endif %}{% endfor %}
|
||||||
151388
checkpoint-302/merges.txt
Normal file
151388
checkpoint-302/merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
checkpoint-302/optimizer.pt
Normal file
3
checkpoint-302/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:0aa25085a7d953219bf1b55fbc4057a1d7f93f43ee341b0ef4ddb28446d1d016
|
||||||
|
size 161816187
|
||||||
3
checkpoint-302/rng_state.pth
Normal file
3
checkpoint-302/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:16f931e11c4d8957d73d287061121ca665ca3d69134cbd0f34b75f951511e096
|
||||||
|
size 14645
|
||||||
3
checkpoint-302/scheduler.pt
Normal file
3
checkpoint-302/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:62d20745a4ca91b8c8b2f758e808c91bb0c71122f057d12f02d187230839a8f9
|
||||||
|
size 1465
|
||||||
31
checkpoint-302/special_tokens_map.json
Normal file
31
checkpoint-302/special_tokens_map.json
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
{
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<|object_ref_start|>",
|
||||||
|
"<|object_ref_end|>",
|
||||||
|
"<|box_start|>",
|
||||||
|
"<|box_end|>",
|
||||||
|
"<|quad_start|>",
|
||||||
|
"<|quad_end|>",
|
||||||
|
"<|vision_start|>",
|
||||||
|
"<|vision_end|>",
|
||||||
|
"<|vision_pad|>",
|
||||||
|
"<|image_pad|>",
|
||||||
|
"<|video_pad|>"
|
||||||
|
],
|
||||||
|
"eos_token": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"pad_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
}
|
||||||
|
}
|
||||||
3
checkpoint-302/tokenizer.json
Normal file
3
checkpoint-302/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
|
||||||
|
size 11421896
|
||||||
208
checkpoint-302/tokenizer_config.json
Normal file
208
checkpoint-302/tokenizer_config.json
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
{
|
||||||
|
"add_bos_token": false,
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"added_tokens_decoder": {
|
||||||
|
"151643": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151644": {
|
||||||
|
"content": "<|im_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151645": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151646": {
|
||||||
|
"content": "<|object_ref_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151647": {
|
||||||
|
"content": "<|object_ref_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151648": {
|
||||||
|
"content": "<|box_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151649": {
|
||||||
|
"content": "<|box_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151650": {
|
||||||
|
"content": "<|quad_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151651": {
|
||||||
|
"content": "<|quad_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151652": {
|
||||||
|
"content": "<|vision_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151653": {
|
||||||
|
"content": "<|vision_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151654": {
|
||||||
|
"content": "<|vision_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151655": {
|
||||||
|
"content": "<|image_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151656": {
|
||||||
|
"content": "<|video_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151657": {
|
||||||
|
"content": "<tool_call>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151658": {
|
||||||
|
"content": "</tool_call>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151659": {
|
||||||
|
"content": "<|fim_prefix|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151660": {
|
||||||
|
"content": "<|fim_middle|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151661": {
|
||||||
|
"content": "<|fim_suffix|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151662": {
|
||||||
|
"content": "<|fim_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151663": {
|
||||||
|
"content": "<|repo_name|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151664": {
|
||||||
|
"content": "<|file_sep|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<|object_ref_start|>",
|
||||||
|
"<|object_ref_end|>",
|
||||||
|
"<|box_start|>",
|
||||||
|
"<|box_end|>",
|
||||||
|
"<|quad_start|>",
|
||||||
|
"<|quad_end|>",
|
||||||
|
"<|vision_start|>",
|
||||||
|
"<|vision_end|>",
|
||||||
|
"<|vision_pad|>",
|
||||||
|
"<|image_pad|>",
|
||||||
|
"<|video_pad|>"
|
||||||
|
],
|
||||||
|
"bos_token": null,
|
||||||
|
"clean_up_tokenization_spaces": false,
|
||||||
|
"eos_token": "<|im_end|>",
|
||||||
|
"errors": "replace",
|
||||||
|
"extra_special_tokens": {},
|
||||||
|
"model_max_length": 131072,
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"padding_side": "left",
|
||||||
|
"split_special_tokens": false,
|
||||||
|
"tokenizer_class": "Qwen2Tokenizer",
|
||||||
|
"unk_token": null
|
||||||
|
}
|
||||||
1
checkpoint-302/tokens_state.
Normal file
1
checkpoint-302/tokens_state.
Normal file
@@ -0,0 +1 @@
|
|||||||
|
{"total": 11104256, "trainable": 9633577}
|
||||||
454
checkpoint-302/trainer_state.json
Normal file
454
checkpoint-302/trainer_state.json
Normal file
@@ -0,0 +1,454 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.1229813664596273,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 302,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.037267080745341616,
|
||||||
|
"grad_norm": 0.19933472573757172,
|
||||||
|
"learning_rate": 2.8125000000000003e-05,
|
||||||
|
"loss": 3.1896,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 24.27871,
|
||||||
|
"step": 10,
|
||||||
|
"tokens/total": 368640,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.07,
|
||||||
|
"tokens/trainable": 321256
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.07453416149068323,
|
||||||
|
"grad_norm": 0.2485538274049759,
|
||||||
|
"learning_rate": 4.9995893860576395e-05,
|
||||||
|
"loss": 3.1828,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 24.11418,
|
||||||
|
"step": 20,
|
||||||
|
"tokens/total": 737280,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.47,
|
||||||
|
"tokens/trainable": 643308
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.11180124223602485,
|
||||||
|
"grad_norm": 0.16610555350780487,
|
||||||
|
"learning_rate": 4.99229333433282e-05,
|
||||||
|
"loss": 3.1795,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 24.03473,
|
||||||
|
"step": 30,
|
||||||
|
"tokens/total": 1105920,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.23,
|
||||||
|
"tokens/trainable": 962499
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.14906832298136646,
|
||||||
|
"grad_norm": 0.16600094735622406,
|
||||||
|
"learning_rate": 4.975903174511846e-05,
|
||||||
|
"loss": 3.1002,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.20239,
|
||||||
|
"step": 40,
|
||||||
|
"tokens/total": 1474560,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.06,
|
||||||
|
"tokens/trainable": 1283109
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.18633540372670807,
|
||||||
|
"grad_norm": 0.17242471873760223,
|
||||||
|
"learning_rate": 4.950478712510446e-05,
|
||||||
|
"loss": 3.0775,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.70407,
|
||||||
|
"step": 50,
|
||||||
|
"tokens/total": 1843200,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.42,
|
||||||
|
"tokens/trainable": 1603821
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2236024844720497,
|
||||||
|
"grad_norm": 0.17550165951251984,
|
||||||
|
"learning_rate": 4.9161127194332655e-05,
|
||||||
|
"loss": 3.1334,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.95188,
|
||||||
|
"step": 60,
|
||||||
|
"tokens/total": 2211840,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.85,
|
||||||
|
"tokens/trainable": 1925471
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2608695652173913,
|
||||||
|
"grad_norm": 0.18663033843040466,
|
||||||
|
"learning_rate": 4.872930593062168e-05,
|
||||||
|
"loss": 3.1937,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 24.37846,
|
||||||
|
"step": 70,
|
||||||
|
"tokens/total": 2580480,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.7,
|
||||||
|
"tokens/trainable": 2242912
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2981366459627329,
|
||||||
|
"grad_norm": 0.20098601281642914,
|
||||||
|
"learning_rate": 4.8210899002933235e-05,
|
||||||
|
"loss": 3.1263,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.7895,
|
||||||
|
"step": 80,
|
||||||
|
"tokens/total": 2949120,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.38,
|
||||||
|
"tokens/trainable": 2562908
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.33540372670807456,
|
||||||
|
"grad_norm": 0.18968884646892548,
|
||||||
|
"learning_rate": 4.760779802192698e-05,
|
||||||
|
"loss": 3.142,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 23.15012,
|
||||||
|
"step": 90,
|
||||||
|
"tokens/total": 3317760,
|
||||||
|
"tokens/train_per_sec_per_gpu": 46.02,
|
||||||
|
"tokens/trainable": 2881497
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.37267080745341613,
|
||||||
|
"grad_norm": 0.21494834125041962,
|
||||||
|
"learning_rate": 4.692220363767833e-05,
|
||||||
|
"loss": 3.1794,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 24.03233,
|
||||||
|
"step": 100,
|
||||||
|
"tokens/total": 3686400,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.95,
|
||||||
|
"tokens/trainable": 3199097
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.40993788819875776,
|
||||||
|
"grad_norm": 0.20388352870941162,
|
||||||
|
"learning_rate": 4.6156617509745015e-05,
|
||||||
|
"loss": 3.1358,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 23.00703,
|
||||||
|
"step": 110,
|
||||||
|
"tokens/total": 4055040,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.96,
|
||||||
|
"tokens/trainable": 3517985
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4472049689440994,
|
||||||
|
"grad_norm": 0.22123299539089203,
|
||||||
|
"learning_rate": 4.531383317888261e-05,
|
||||||
|
"loss": 3.1032,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.2691,
|
||||||
|
"step": 120,
|
||||||
|
"tokens/total": 4423680,
|
||||||
|
"tokens/train_per_sec_per_gpu": 41.16,
|
||||||
|
"tokens/trainable": 3837324
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.484472049689441,
|
||||||
|
"grad_norm": 0.2072950154542923,
|
||||||
|
"learning_rate": 4.439692587371731e-05,
|
||||||
|
"loss": 3.1074,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.36283,
|
||||||
|
"step": 130,
|
||||||
|
"tokens/total": 4792320,
|
||||||
|
"tokens/train_per_sec_per_gpu": 46.1,
|
||||||
|
"tokens/trainable": 4159485
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5217391304347826,
|
||||||
|
"grad_norm": 0.22470185160636902,
|
||||||
|
"learning_rate": 4.3409241289570156e-05,
|
||||||
|
"loss": 3.0826,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.81505,
|
||||||
|
"step": 140,
|
||||||
|
"tokens/total": 5160960,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.04,
|
||||||
|
"tokens/trainable": 4479023
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5590062111801242,
|
||||||
|
"grad_norm": 0.23675978183746338,
|
||||||
|
"learning_rate": 4.235438338037764e-05,
|
||||||
|
"loss": 3.0895,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.96609,
|
||||||
|
"step": 150,
|
||||||
|
"tokens/total": 5529600,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.54,
|
||||||
|
"tokens/trainable": 4798599
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5962732919254659,
|
||||||
|
"grad_norm": 0.22080311179161072,
|
||||||
|
"learning_rate": 4.123620120825459e-05,
|
||||||
|
"loss": 3.0991,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.17798,
|
||||||
|
"step": 160,
|
||||||
|
"tokens/total": 5898240,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.18,
|
||||||
|
"tokens/trainable": 5117093
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6335403726708074,
|
||||||
|
"grad_norm": 0.22512708604335785,
|
||||||
|
"learning_rate": 4.005877489868401e-05,
|
||||||
|
"loss": 3.1219,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.68945,
|
||||||
|
"step": 170,
|
||||||
|
"tokens/total": 6266880,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.85,
|
||||||
|
"tokens/trainable": 5436542
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6708074534161491,
|
||||||
|
"grad_norm": 0.22725142538547516,
|
||||||
|
"learning_rate": 3.88264007525816e-05,
|
||||||
|
"loss": 3.167,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 23.73617,
|
||||||
|
"step": 180,
|
||||||
|
"tokens/total": 6635520,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.1,
|
||||||
|
"tokens/trainable": 5757152
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7080745341614907,
|
||||||
|
"grad_norm": 0.23717202246189117,
|
||||||
|
"learning_rate": 3.754357556955966e-05,
|
||||||
|
"loss": 3.1361,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 23.01394,
|
||||||
|
"step": 190,
|
||||||
|
"tokens/total": 7004160,
|
||||||
|
"tokens/train_per_sec_per_gpu": 42.08,
|
||||||
|
"tokens/trainable": 6076610
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7453416149068323,
|
||||||
|
"grad_norm": 0.2373926043510437,
|
||||||
|
"learning_rate": 3.621498023959294e-05,
|
||||||
|
"loss": 3.1262,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.78722,
|
||||||
|
"step": 200,
|
||||||
|
"tokens/total": 7372800,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.68,
|
||||||
|
"tokens/trainable": 6396405
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.782608695652174,
|
||||||
|
"grad_norm": 0.22468852996826172,
|
||||||
|
"learning_rate": 3.484546266295883e-05,
|
||||||
|
"loss": 3.1138,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.50641,
|
||||||
|
"step": 210,
|
||||||
|
"tokens/total": 7741440,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.14,
|
||||||
|
"tokens/trainable": 6714000
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8198757763975155,
|
||||||
|
"grad_norm": 0.23512248694896698,
|
||||||
|
"learning_rate": 3.344002006077487e-05,
|
||||||
|
"loss": 3.113,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.48841,
|
||||||
|
"step": 220,
|
||||||
|
"tokens/total": 8110080,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.86,
|
||||||
|
"tokens/trainable": 7034597
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8571428571428571,
|
||||||
|
"grad_norm": 0.23428736627101898,
|
||||||
|
"learning_rate": 3.200378074068092e-05,
|
||||||
|
"loss": 3.0454,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.01844,
|
||||||
|
"step": 230,
|
||||||
|
"tokens/total": 8478720,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.72,
|
||||||
|
"tokens/trainable": 7357866
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8944099378881988,
|
||||||
|
"grad_norm": 0.25676628947257996,
|
||||||
|
"learning_rate": 3.0541985384200686e-05,
|
||||||
|
"loss": 3.1156,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.54695,
|
||||||
|
"step": 240,
|
||||||
|
"tokens/total": 8847360,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.71,
|
||||||
|
"tokens/trainable": 7675991
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9316770186335404,
|
||||||
|
"grad_norm": 0.2449398934841156,
|
||||||
|
"learning_rate": 2.9059967924063153e-05,
|
||||||
|
"loss": 3.1204,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.65544,
|
||||||
|
"step": 250,
|
||||||
|
"tokens/total": 9216000,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.14,
|
||||||
|
"tokens/trainable": 7995656
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.968944099378882,
|
||||||
|
"grad_norm": 0.2541828453540802,
|
||||||
|
"learning_rate": 2.7563136081260582e-05,
|
||||||
|
"loss": 3.1261,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.78494,
|
||||||
|
"step": 260,
|
||||||
|
"tokens/total": 9584640,
|
||||||
|
"tokens/train_per_sec_per_gpu": 42.8,
|
||||||
|
"tokens/trainable": 8314538
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0037267080745342,
|
||||||
|
"grad_norm": 0.24245300889015198,
|
||||||
|
"learning_rate": 2.6056951632860976e-05,
|
||||||
|
"loss": 3.0867,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.90467,
|
||||||
|
"step": 270,
|
||||||
|
"tokens/total": 9924608,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.48,
|
||||||
|
"tokens/trainable": 8609040
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0409937888198757,
|
||||||
|
"grad_norm": 0.2593900263309479,
|
||||||
|
"learning_rate": 2.4546910482575908e-05,
|
||||||
|
"loss": 3.0741,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.63041,
|
||||||
|
"step": 280,
|
||||||
|
"tokens/total": 10293248,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.2,
|
||||||
|
"tokens/trainable": 8927894
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0782608695652174,
|
||||||
|
"grad_norm": 0.2553490996360779,
|
||||||
|
"learning_rate": 2.303852260680388e-05,
|
||||||
|
"loss": 3.0859,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.88716,
|
||||||
|
"step": 290,
|
||||||
|
"tokens/total": 10661888,
|
||||||
|
"tokens/train_per_sec_per_gpu": 46.62,
|
||||||
|
"tokens/trainable": 9250881
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.115527950310559,
|
||||||
|
"grad_norm": 0.2889447510242462,
|
||||||
|
"learning_rate": 2.1537291949324007e-05,
|
||||||
|
"loss": 3.0993,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.18242,
|
||||||
|
"step": 300,
|
||||||
|
"tokens/total": 11030528,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.46,
|
||||||
|
"tokens/trainable": 9569197
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 10,
|
||||||
|
"max_steps": 536,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 302,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 4.7242863209506406e+17,
|
||||||
|
"train_batch_size": 6,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-302/training_args.bin
Normal file
3
checkpoint-302/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a4e3f01ef14d3ae1fce2d05266687ff3783968fc7afb23454eabe24a96b85868
|
||||||
|
size 7313
|
||||||
1
checkpoint-302/vocab.json
Normal file
1
checkpoint-302/vocab.json
Normal file
File diff suppressed because one or more lines are too long
208
checkpoint-536/README.md
Normal file
208
checkpoint-536/README.md
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
---
|
||||||
|
base_model: Orion-zhen/Meissa-Qwen2.5-7B-Instruct
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- axolotl
|
||||||
|
- base_model:adapter:Orion-zhen/Meissa-Qwen2.5-7B-Instruct
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.18.1
|
||||||
46
checkpoint-536/adapter_config.json
Normal file
46
checkpoint-536/adapter_config.json
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Orion-zhen/Meissa-Qwen2.5-7B-Instruct",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": null,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 32,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.1,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.18.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 8,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"o_proj",
|
||||||
|
"k_proj",
|
||||||
|
"q_proj",
|
||||||
|
"up_proj",
|
||||||
|
"down_proj",
|
||||||
|
"gate_proj",
|
||||||
|
"v_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": [],
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-536/adapter_model.safetensors
Normal file
3
checkpoint-536/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f0380466a97b625606644758c2302e378fb50d91b23bdffa6dad1121c0439a23
|
||||||
|
size 80792096
|
||||||
24
checkpoint-536/added_tokens.json
Normal file
24
checkpoint-536/added_tokens.json
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
{
|
||||||
|
"</tool_call>": 151658,
|
||||||
|
"<tool_call>": 151657,
|
||||||
|
"<|box_end|>": 151649,
|
||||||
|
"<|box_start|>": 151648,
|
||||||
|
"<|endoftext|>": 151643,
|
||||||
|
"<|file_sep|>": 151664,
|
||||||
|
"<|fim_middle|>": 151660,
|
||||||
|
"<|fim_pad|>": 151662,
|
||||||
|
"<|fim_prefix|>": 151659,
|
||||||
|
"<|fim_suffix|>": 151661,
|
||||||
|
"<|im_end|>": 151645,
|
||||||
|
"<|im_start|>": 151644,
|
||||||
|
"<|image_pad|>": 151655,
|
||||||
|
"<|object_ref_end|>": 151647,
|
||||||
|
"<|object_ref_start|>": 151646,
|
||||||
|
"<|quad_end|>": 151651,
|
||||||
|
"<|quad_start|>": 151650,
|
||||||
|
"<|repo_name|>": 151663,
|
||||||
|
"<|video_pad|>": 151656,
|
||||||
|
"<|vision_end|>": 151653,
|
||||||
|
"<|vision_pad|>": 151654,
|
||||||
|
"<|vision_start|>": 151652
|
||||||
|
}
|
||||||
7
checkpoint-536/chat_template.jinja
Normal file
7
checkpoint-536/chat_template.jinja
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system
|
||||||
|
' + system_message + '<|im_end|>
|
||||||
|
' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user
|
||||||
|
' + content + '<|im_end|>
|
||||||
|
<|im_start|>assistant
|
||||||
|
' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '
|
||||||
|
' }}{% endif %}{% endfor %}
|
||||||
151388
checkpoint-536/merges.txt
Normal file
151388
checkpoint-536/merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
checkpoint-536/optimizer.pt
Normal file
3
checkpoint-536/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f3b88877f50289c3bc1b1460000e025a7f8710cb90fb4268434889c86840d563
|
||||||
|
size 161816187
|
||||||
3
checkpoint-536/rng_state.pth
Normal file
3
checkpoint-536/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f379b12b68960eb464c33d8d294793c4a0e5a97a6e63a22a587c9c71d71e09db
|
||||||
|
size 14645
|
||||||
3
checkpoint-536/scheduler.pt
Normal file
3
checkpoint-536/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:eeca59cdcfd3cf609ccdfc78110905e3ffbe0ec3c19d0f2c427c401666851552
|
||||||
|
size 1465
|
||||||
31
checkpoint-536/special_tokens_map.json
Normal file
31
checkpoint-536/special_tokens_map.json
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
{
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<|object_ref_start|>",
|
||||||
|
"<|object_ref_end|>",
|
||||||
|
"<|box_start|>",
|
||||||
|
"<|box_end|>",
|
||||||
|
"<|quad_start|>",
|
||||||
|
"<|quad_end|>",
|
||||||
|
"<|vision_start|>",
|
||||||
|
"<|vision_end|>",
|
||||||
|
"<|vision_pad|>",
|
||||||
|
"<|image_pad|>",
|
||||||
|
"<|video_pad|>"
|
||||||
|
],
|
||||||
|
"eos_token": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"pad_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
}
|
||||||
|
}
|
||||||
3
checkpoint-536/tokenizer.json
Normal file
3
checkpoint-536/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
|
||||||
|
size 11421896
|
||||||
208
checkpoint-536/tokenizer_config.json
Normal file
208
checkpoint-536/tokenizer_config.json
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
{
|
||||||
|
"add_bos_token": false,
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"added_tokens_decoder": {
|
||||||
|
"151643": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151644": {
|
||||||
|
"content": "<|im_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151645": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151646": {
|
||||||
|
"content": "<|object_ref_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151647": {
|
||||||
|
"content": "<|object_ref_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151648": {
|
||||||
|
"content": "<|box_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151649": {
|
||||||
|
"content": "<|box_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151650": {
|
||||||
|
"content": "<|quad_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151651": {
|
||||||
|
"content": "<|quad_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151652": {
|
||||||
|
"content": "<|vision_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151653": {
|
||||||
|
"content": "<|vision_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151654": {
|
||||||
|
"content": "<|vision_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151655": {
|
||||||
|
"content": "<|image_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151656": {
|
||||||
|
"content": "<|video_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151657": {
|
||||||
|
"content": "<tool_call>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151658": {
|
||||||
|
"content": "</tool_call>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151659": {
|
||||||
|
"content": "<|fim_prefix|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151660": {
|
||||||
|
"content": "<|fim_middle|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151661": {
|
||||||
|
"content": "<|fim_suffix|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151662": {
|
||||||
|
"content": "<|fim_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151663": {
|
||||||
|
"content": "<|repo_name|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151664": {
|
||||||
|
"content": "<|file_sep|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<|object_ref_start|>",
|
||||||
|
"<|object_ref_end|>",
|
||||||
|
"<|box_start|>",
|
||||||
|
"<|box_end|>",
|
||||||
|
"<|quad_start|>",
|
||||||
|
"<|quad_end|>",
|
||||||
|
"<|vision_start|>",
|
||||||
|
"<|vision_end|>",
|
||||||
|
"<|vision_pad|>",
|
||||||
|
"<|image_pad|>",
|
||||||
|
"<|video_pad|>"
|
||||||
|
],
|
||||||
|
"bos_token": null,
|
||||||
|
"clean_up_tokenization_spaces": false,
|
||||||
|
"eos_token": "<|im_end|>",
|
||||||
|
"errors": "replace",
|
||||||
|
"extra_special_tokens": {},
|
||||||
|
"model_max_length": 131072,
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"padding_side": "left",
|
||||||
|
"split_special_tokens": false,
|
||||||
|
"tokenizer_class": "Qwen2Tokenizer",
|
||||||
|
"unk_token": null
|
||||||
|
}
|
||||||
1
checkpoint-536/tokens_state.
Normal file
1
checkpoint-536/tokens_state.
Normal file
@@ -0,0 +1 @@
|
|||||||
|
{"total": 19730432, "trainable": 17115300}
|
||||||
776
checkpoint-536/trainer_state.json
Normal file
776
checkpoint-536/trainer_state.json
Normal file
@@ -0,0 +1,776 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.995031055900621,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 536,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.037267080745341616,
|
||||||
|
"grad_norm": 0.19933472573757172,
|
||||||
|
"learning_rate": 2.8125000000000003e-05,
|
||||||
|
"loss": 3.1896,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 24.27871,
|
||||||
|
"step": 10,
|
||||||
|
"tokens/total": 368640,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.07,
|
||||||
|
"tokens/trainable": 321256
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.07453416149068323,
|
||||||
|
"grad_norm": 0.2485538274049759,
|
||||||
|
"learning_rate": 4.9995893860576395e-05,
|
||||||
|
"loss": 3.1828,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 24.11418,
|
||||||
|
"step": 20,
|
||||||
|
"tokens/total": 737280,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.47,
|
||||||
|
"tokens/trainable": 643308
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.11180124223602485,
|
||||||
|
"grad_norm": 0.16610555350780487,
|
||||||
|
"learning_rate": 4.99229333433282e-05,
|
||||||
|
"loss": 3.1795,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 24.03473,
|
||||||
|
"step": 30,
|
||||||
|
"tokens/total": 1105920,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.23,
|
||||||
|
"tokens/trainable": 962499
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.14906832298136646,
|
||||||
|
"grad_norm": 0.16600094735622406,
|
||||||
|
"learning_rate": 4.975903174511846e-05,
|
||||||
|
"loss": 3.1002,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.20239,
|
||||||
|
"step": 40,
|
||||||
|
"tokens/total": 1474560,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.06,
|
||||||
|
"tokens/trainable": 1283109
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.18633540372670807,
|
||||||
|
"grad_norm": 0.17242471873760223,
|
||||||
|
"learning_rate": 4.950478712510446e-05,
|
||||||
|
"loss": 3.0775,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.70407,
|
||||||
|
"step": 50,
|
||||||
|
"tokens/total": 1843200,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.42,
|
||||||
|
"tokens/trainable": 1603821
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2236024844720497,
|
||||||
|
"grad_norm": 0.17550165951251984,
|
||||||
|
"learning_rate": 4.9161127194332655e-05,
|
||||||
|
"loss": 3.1334,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.95188,
|
||||||
|
"step": 60,
|
||||||
|
"tokens/total": 2211840,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.85,
|
||||||
|
"tokens/trainable": 1925471
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2608695652173913,
|
||||||
|
"grad_norm": 0.18663033843040466,
|
||||||
|
"learning_rate": 4.872930593062168e-05,
|
||||||
|
"loss": 3.1937,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 24.37846,
|
||||||
|
"step": 70,
|
||||||
|
"tokens/total": 2580480,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.7,
|
||||||
|
"tokens/trainable": 2242912
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2981366459627329,
|
||||||
|
"grad_norm": 0.20098601281642914,
|
||||||
|
"learning_rate": 4.8210899002933235e-05,
|
||||||
|
"loss": 3.1263,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.7895,
|
||||||
|
"step": 80,
|
||||||
|
"tokens/total": 2949120,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.38,
|
||||||
|
"tokens/trainable": 2562908
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.33540372670807456,
|
||||||
|
"grad_norm": 0.18968884646892548,
|
||||||
|
"learning_rate": 4.760779802192698e-05,
|
||||||
|
"loss": 3.142,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 23.15012,
|
||||||
|
"step": 90,
|
||||||
|
"tokens/total": 3317760,
|
||||||
|
"tokens/train_per_sec_per_gpu": 46.02,
|
||||||
|
"tokens/trainable": 2881497
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.37267080745341613,
|
||||||
|
"grad_norm": 0.21494834125041962,
|
||||||
|
"learning_rate": 4.692220363767833e-05,
|
||||||
|
"loss": 3.1794,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 24.03233,
|
||||||
|
"step": 100,
|
||||||
|
"tokens/total": 3686400,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.95,
|
||||||
|
"tokens/trainable": 3199097
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.40993788819875776,
|
||||||
|
"grad_norm": 0.20388352870941162,
|
||||||
|
"learning_rate": 4.6156617509745015e-05,
|
||||||
|
"loss": 3.1358,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 23.00703,
|
||||||
|
"step": 110,
|
||||||
|
"tokens/total": 4055040,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.96,
|
||||||
|
"tokens/trainable": 3517985
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4472049689440994,
|
||||||
|
"grad_norm": 0.22123299539089203,
|
||||||
|
"learning_rate": 4.531383317888261e-05,
|
||||||
|
"loss": 3.1032,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.2691,
|
||||||
|
"step": 120,
|
||||||
|
"tokens/total": 4423680,
|
||||||
|
"tokens/train_per_sec_per_gpu": 41.16,
|
||||||
|
"tokens/trainable": 3837324
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.484472049689441,
|
||||||
|
"grad_norm": 0.2072950154542923,
|
||||||
|
"learning_rate": 4.439692587371731e-05,
|
||||||
|
"loss": 3.1074,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.36283,
|
||||||
|
"step": 130,
|
||||||
|
"tokens/total": 4792320,
|
||||||
|
"tokens/train_per_sec_per_gpu": 46.1,
|
||||||
|
"tokens/trainable": 4159485
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5217391304347826,
|
||||||
|
"grad_norm": 0.22470185160636902,
|
||||||
|
"learning_rate": 4.3409241289570156e-05,
|
||||||
|
"loss": 3.0826,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.81505,
|
||||||
|
"step": 140,
|
||||||
|
"tokens/total": 5160960,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.04,
|
||||||
|
"tokens/trainable": 4479023
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5590062111801242,
|
||||||
|
"grad_norm": 0.23675978183746338,
|
||||||
|
"learning_rate": 4.235438338037764e-05,
|
||||||
|
"loss": 3.0895,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.96609,
|
||||||
|
"step": 150,
|
||||||
|
"tokens/total": 5529600,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.54,
|
||||||
|
"tokens/trainable": 4798599
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5962732919254659,
|
||||||
|
"grad_norm": 0.22080311179161072,
|
||||||
|
"learning_rate": 4.123620120825459e-05,
|
||||||
|
"loss": 3.0991,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.17798,
|
||||||
|
"step": 160,
|
||||||
|
"tokens/total": 5898240,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.18,
|
||||||
|
"tokens/trainable": 5117093
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6335403726708074,
|
||||||
|
"grad_norm": 0.22512708604335785,
|
||||||
|
"learning_rate": 4.005877489868401e-05,
|
||||||
|
"loss": 3.1219,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.68945,
|
||||||
|
"step": 170,
|
||||||
|
"tokens/total": 6266880,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.85,
|
||||||
|
"tokens/trainable": 5436542
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6708074534161491,
|
||||||
|
"grad_norm": 0.22725142538547516,
|
||||||
|
"learning_rate": 3.88264007525816e-05,
|
||||||
|
"loss": 3.167,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 23.73617,
|
||||||
|
"step": 180,
|
||||||
|
"tokens/total": 6635520,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.1,
|
||||||
|
"tokens/trainable": 5757152
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7080745341614907,
|
||||||
|
"grad_norm": 0.23717202246189117,
|
||||||
|
"learning_rate": 3.754357556955966e-05,
|
||||||
|
"loss": 3.1361,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 23.01394,
|
||||||
|
"step": 190,
|
||||||
|
"tokens/total": 7004160,
|
||||||
|
"tokens/train_per_sec_per_gpu": 42.08,
|
||||||
|
"tokens/trainable": 6076610
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7453416149068323,
|
||||||
|
"grad_norm": 0.2373926043510437,
|
||||||
|
"learning_rate": 3.621498023959294e-05,
|
||||||
|
"loss": 3.1262,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.78722,
|
||||||
|
"step": 200,
|
||||||
|
"tokens/total": 7372800,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.68,
|
||||||
|
"tokens/trainable": 6396405
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.782608695652174,
|
||||||
|
"grad_norm": 0.22468852996826172,
|
||||||
|
"learning_rate": 3.484546266295883e-05,
|
||||||
|
"loss": 3.1138,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.50641,
|
||||||
|
"step": 210,
|
||||||
|
"tokens/total": 7741440,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.14,
|
||||||
|
"tokens/trainable": 6714000
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8198757763975155,
|
||||||
|
"grad_norm": 0.23512248694896698,
|
||||||
|
"learning_rate": 3.344002006077487e-05,
|
||||||
|
"loss": 3.113,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.48841,
|
||||||
|
"step": 220,
|
||||||
|
"tokens/total": 8110080,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.86,
|
||||||
|
"tokens/trainable": 7034597
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8571428571428571,
|
||||||
|
"grad_norm": 0.23428736627101898,
|
||||||
|
"learning_rate": 3.200378074068092e-05,
|
||||||
|
"loss": 3.0454,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.01844,
|
||||||
|
"step": 230,
|
||||||
|
"tokens/total": 8478720,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.72,
|
||||||
|
"tokens/trainable": 7357866
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8944099378881988,
|
||||||
|
"grad_norm": 0.25676628947257996,
|
||||||
|
"learning_rate": 3.0541985384200686e-05,
|
||||||
|
"loss": 3.1156,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.54695,
|
||||||
|
"step": 240,
|
||||||
|
"tokens/total": 8847360,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.71,
|
||||||
|
"tokens/trainable": 7675991
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9316770186335404,
|
||||||
|
"grad_norm": 0.2449398934841156,
|
||||||
|
"learning_rate": 2.9059967924063153e-05,
|
||||||
|
"loss": 3.1204,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.65544,
|
||||||
|
"step": 250,
|
||||||
|
"tokens/total": 9216000,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.14,
|
||||||
|
"tokens/trainable": 7995656
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.968944099378882,
|
||||||
|
"grad_norm": 0.2541828453540802,
|
||||||
|
"learning_rate": 2.7563136081260582e-05,
|
||||||
|
"loss": 3.1261,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.78494,
|
||||||
|
"step": 260,
|
||||||
|
"tokens/total": 9584640,
|
||||||
|
"tokens/train_per_sec_per_gpu": 42.8,
|
||||||
|
"tokens/trainable": 8314538
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0037267080745342,
|
||||||
|
"grad_norm": 0.24245300889015198,
|
||||||
|
"learning_rate": 2.6056951632860976e-05,
|
||||||
|
"loss": 3.0867,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.90467,
|
||||||
|
"step": 270,
|
||||||
|
"tokens/total": 9924608,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.48,
|
||||||
|
"tokens/trainable": 8609040
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0409937888198757,
|
||||||
|
"grad_norm": 0.2593900263309479,
|
||||||
|
"learning_rate": 2.4546910482575908e-05,
|
||||||
|
"loss": 3.0741,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.63041,
|
||||||
|
"step": 280,
|
||||||
|
"tokens/total": 10293248,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.2,
|
||||||
|
"tokens/trainable": 8927894
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0782608695652174,
|
||||||
|
"grad_norm": 0.2553490996360779,
|
||||||
|
"learning_rate": 2.303852260680388e-05,
|
||||||
|
"loss": 3.0859,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.88716,
|
||||||
|
"step": 290,
|
||||||
|
"tokens/total": 10661888,
|
||||||
|
"tokens/train_per_sec_per_gpu": 46.62,
|
||||||
|
"tokens/trainable": 9250881
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.115527950310559,
|
||||||
|
"grad_norm": 0.2889447510242462,
|
||||||
|
"learning_rate": 2.1537291949324007e-05,
|
||||||
|
"loss": 3.0993,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.18242,
|
||||||
|
"step": 300,
|
||||||
|
"tokens/total": 11030528,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.46,
|
||||||
|
"tokens/trainable": 9569197
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1527950310559005,
|
||||||
|
"grad_norm": 0.3012157380580902,
|
||||||
|
"learning_rate": 2.0048696338002015e-05,
|
||||||
|
"loss": 3.1172,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.58306,
|
||||||
|
"step": 310,
|
||||||
|
"tokens/total": 11399168,
|
||||||
|
"tokens/train_per_sec_per_gpu": 42.07,
|
||||||
|
"tokens/trainable": 9889854
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1900621118012422,
|
||||||
|
"grad_norm": 0.290158212184906,
|
||||||
|
"learning_rate": 1.857816749679043e-05,
|
||||||
|
"loss": 3.1,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.19795,
|
||||||
|
"step": 320,
|
||||||
|
"tokens/total": 11767808,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.72,
|
||||||
|
"tokens/trainable": 10207408
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.227329192546584,
|
||||||
|
"grad_norm": 0.315685898065567,
|
||||||
|
"learning_rate": 1.713107122595693e-05,
|
||||||
|
"loss": 3.0941,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.06737,
|
||||||
|
"step": 330,
|
||||||
|
"tokens/total": 12136448,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.89,
|
||||||
|
"tokens/trainable": 10527550
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2645962732919256,
|
||||||
|
"grad_norm": 0.29844436049461365,
|
||||||
|
"learning_rate": 1.571268782286107e-05,
|
||||||
|
"loss": 3.0997,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.19129,
|
||||||
|
"step": 340,
|
||||||
|
"tokens/total": 12505088,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.94,
|
||||||
|
"tokens/trainable": 10848911
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.301863354037267,
|
||||||
|
"grad_norm": 0.29774603247642517,
|
||||||
|
"learning_rate": 1.4328192814721846e-05,
|
||||||
|
"loss": 3.0617,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.36384,
|
||||||
|
"step": 350,
|
||||||
|
"tokens/total": 12873728,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.22,
|
||||||
|
"tokens/trainable": 11167329
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.3391304347826087,
|
||||||
|
"grad_norm": 0.305044025182724,
|
||||||
|
"learning_rate": 1.2982638073680112e-05,
|
||||||
|
"loss": 3.0857,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.88278,
|
||||||
|
"step": 360,
|
||||||
|
"tokens/total": 13242368,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.71,
|
||||||
|
"tokens/trainable": 11489331
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.3763975155279504,
|
||||||
|
"grad_norm": 0.30985885858535767,
|
||||||
|
"learning_rate": 1.1680933383065057e-05,
|
||||||
|
"loss": 3.0768,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.68889,
|
||||||
|
"step": 370,
|
||||||
|
"tokens/total": 13611008,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.03,
|
||||||
|
"tokens/trainable": 11809342
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.4136645962732919,
|
||||||
|
"grad_norm": 0.31623899936676025,
|
||||||
|
"learning_rate": 1.042782852212725e-05,
|
||||||
|
"loss": 3.0995,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.18686,
|
||||||
|
"step": 380,
|
||||||
|
"tokens/total": 13979648,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.18,
|
||||||
|
"tokens/trainable": 12127653
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.4509316770186336,
|
||||||
|
"grad_norm": 0.31580615043640137,
|
||||||
|
"learning_rate": 9.227895934609235e-06,
|
||||||
|
"loss": 3.0674,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.48597,
|
||||||
|
"step": 390,
|
||||||
|
"tokens/total": 14348288,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.49,
|
||||||
|
"tokens/trainable": 12446447
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.4881987577639753,
|
||||||
|
"grad_norm": 0.3268173635005951,
|
||||||
|
"learning_rate": 8.085514044394216e-06,
|
||||||
|
"loss": 3.1286,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.84198,
|
||||||
|
"step": 400,
|
||||||
|
"tokens/total": 14716928,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.11,
|
||||||
|
"tokens/trainable": 12766899
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.5254658385093167,
|
||||||
|
"grad_norm": 0.31086301803588867,
|
||||||
|
"learning_rate": 7.004851279112129e-06,
|
||||||
|
"loss": 3.0784,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.72362,
|
||||||
|
"step": 410,
|
||||||
|
"tokens/total": 15085568,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.54,
|
||||||
|
"tokens/trainable": 13088335
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.5627329192546584,
|
||||||
|
"grad_norm": 0.3192853629589081,
|
||||||
|
"learning_rate": 5.989850859999227e-06,
|
||||||
|
"loss": 3.0708,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.55914,
|
||||||
|
"step": 420,
|
||||||
|
"tokens/total": 15454208,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.24,
|
||||||
|
"tokens/trainable": 13408321
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.6,
|
||||||
|
"grad_norm": 0.3102209270000458,
|
||||||
|
"learning_rate": 5.044216413511494e-06,
|
||||||
|
"loss": 3.0687,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.51392,
|
||||||
|
"step": 430,
|
||||||
|
"tokens/total": 15822848,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.8,
|
||||||
|
"tokens/trainable": 13731661
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.6372670807453416,
|
||||||
|
"grad_norm": 0.3230491876602173,
|
||||||
|
"learning_rate": 4.171398457193274e-06,
|
||||||
|
"loss": 3.0735,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.61743,
|
||||||
|
"step": 440,
|
||||||
|
"tokens/total": 16191488,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.26,
|
||||||
|
"tokens/trainable": 14052955
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.6745341614906832,
|
||||||
|
"grad_norm": 0.3240269124507904,
|
||||||
|
"learning_rate": 3.3745818091130033e-06,
|
||||||
|
"loss": 3.1082,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.38072,
|
||||||
|
"step": 450,
|
||||||
|
"tokens/total": 16560128,
|
||||||
|
"tokens/train_per_sec_per_gpu": 43.83,
|
||||||
|
"tokens/trainable": 14372424
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.711801242236025,
|
||||||
|
"grad_norm": 0.3217725157737732,
|
||||||
|
"learning_rate": 2.6566739668074576e-06,
|
||||||
|
"loss": 3.1315,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.90832,
|
||||||
|
"step": 460,
|
||||||
|
"tokens/total": 16928768,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.5,
|
||||||
|
"tokens/trainable": 14690565
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.7490683229813664,
|
||||||
|
"grad_norm": 0.3098929524421692,
|
||||||
|
"learning_rate": 2.020294498138503e-06,
|
||||||
|
"loss": 3.066,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.45591,
|
||||||
|
"step": 470,
|
||||||
|
"tokens/total": 17297408,
|
||||||
|
"tokens/train_per_sec_per_gpu": 45.19,
|
||||||
|
"tokens/trainable": 15008561
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.786335403726708,
|
||||||
|
"grad_norm": 0.3308022618293762,
|
||||||
|
"learning_rate": 1.4677654827741111e-06,
|
||||||
|
"loss": 3.1263,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.7895,
|
||||||
|
"step": 480,
|
||||||
|
"tokens/total": 17666048,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.69,
|
||||||
|
"tokens/trainable": 15325346
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.8236024844720498,
|
||||||
|
"grad_norm": 0.33477458357810974,
|
||||||
|
"learning_rate": 1.0011030391714278e-06,
|
||||||
|
"loss": 3.0856,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.88059,
|
||||||
|
"step": 490,
|
||||||
|
"tokens/total": 18034688,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.1,
|
||||||
|
"tokens/trainable": 15644995
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.8608695652173912,
|
||||||
|
"grad_norm": 0.3346908390522003,
|
||||||
|
"learning_rate": 6.220099679791897e-07,
|
||||||
|
"loss": 3.1043,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.29361,
|
||||||
|
"step": 500,
|
||||||
|
"tokens/total": 18403328,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.87,
|
||||||
|
"tokens/trainable": 15966250
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.898136645962733,
|
||||||
|
"grad_norm": 0.3268662095069885,
|
||||||
|
"learning_rate": 3.3186953870281144e-07,
|
||||||
|
"loss": 3.0731,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 21.60879,
|
||||||
|
"step": 510,
|
||||||
|
"tokens/total": 18771968,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.17,
|
||||||
|
"tokens/trainable": 16285364
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.9354037267080746,
|
||||||
|
"grad_norm": 0.32842233777046204,
|
||||||
|
"learning_rate": 1.3174044230396532e-07,
|
||||||
|
"loss": 3.1173,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.58532,
|
||||||
|
"step": 520,
|
||||||
|
"tokens/total": 19140608,
|
||||||
|
"tokens/train_per_sec_per_gpu": 42.71,
|
||||||
|
"tokens/trainable": 16603401
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.972670807453416,
|
||||||
|
"grad_norm": 0.33898642659187317,
|
||||||
|
"learning_rate": 2.2352928152089003e-08,
|
||||||
|
"loss": 3.1123,
|
||||||
|
"memory/device_reserved (GiB)": 45.16,
|
||||||
|
"memory/max_active (GiB)": 37.89,
|
||||||
|
"memory/max_allocated (GiB)": 37.89,
|
||||||
|
"ppl": 22.47267,
|
||||||
|
"step": 530,
|
||||||
|
"tokens/total": 19509248,
|
||||||
|
"tokens/train_per_sec_per_gpu": 44.84,
|
||||||
|
"tokens/trainable": 16923422
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 10,
|
||||||
|
"max_steps": 536,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 302,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": true
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 8.394277834016686e+17,
|
||||||
|
"train_batch_size": 6,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-536/training_args.bin
Normal file
3
checkpoint-536/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a4e3f01ef14d3ae1fce2d05266687ff3783968fc7afb23454eabe24a96b85868
|
||||||
|
size 7313
|
||||||
1
checkpoint-536/vocab.json
Normal file
1
checkpoint-536/vocab.json
Normal file
File diff suppressed because one or more lines are too long
57
config.json
Normal file
57
config.json
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"Qwen2ForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"dtype": "bfloat16",
|
||||||
|
"eos_token_id": 151645,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 3584,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 18944,
|
||||||
|
"layer_types": [
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention"
|
||||||
|
],
|
||||||
|
"max_position_embeddings": 32768,
|
||||||
|
"max_window_layers": 28,
|
||||||
|
"model_type": "qwen2",
|
||||||
|
"num_attention_heads": 28,
|
||||||
|
"num_hidden_layers": 28,
|
||||||
|
"num_key_value_heads": 4,
|
||||||
|
"rms_norm_eps": 1e-06,
|
||||||
|
"rope_scaling": null,
|
||||||
|
"rope_theta": 1000000.0,
|
||||||
|
"sliding_window": null,
|
||||||
|
"tie_word_embeddings": false,
|
||||||
|
"transformers_version": "4.57.1",
|
||||||
|
"use_cache": false,
|
||||||
|
"use_sliding_window": false,
|
||||||
|
"vocab_size": 152064
|
||||||
|
}
|
||||||
165
debug.log
Normal file
165
debug.log
Normal file
@@ -0,0 +1,165 @@
|
|||||||
|
[2026-01-13 12:24:55,936] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:4815] baseline 0.000GB ()
|
||||||
|
[2026-01-13 12:24:55,937] [INFO] [axolotl.cli.config.load_cfg:259] [PID:4815] config:
|
||||||
|
{
|
||||||
|
"activation_offloading": false,
|
||||||
|
"adapter": "lora",
|
||||||
|
"axolotl_config_path": "zhihu_sft.yml",
|
||||||
|
"base_model": "Orion-zhen/Meissa-Qwen2.5-7B-Instruct",
|
||||||
|
"base_model_config": "Orion-zhen/Meissa-Qwen2.5-7B-Instruct",
|
||||||
|
"batch_size": 18,
|
||||||
|
"bf16": true,
|
||||||
|
"capabilities": {
|
||||||
|
"bf16": true,
|
||||||
|
"compute_capability": "sm_89",
|
||||||
|
"fp8": false,
|
||||||
|
"n_gpu": 1,
|
||||||
|
"n_node": 1
|
||||||
|
},
|
||||||
|
"context_parallel_size": 1,
|
||||||
|
"dataloader_num_workers": 1,
|
||||||
|
"dataloader_pin_memory": true,
|
||||||
|
"dataloader_prefetch_factor": 256,
|
||||||
|
"dataset_num_proc": 32,
|
||||||
|
"datasets": [
|
||||||
|
{
|
||||||
|
"chat_template": "tokenizer_default",
|
||||||
|
"message_property_mappings": {
|
||||||
|
"content": "content",
|
||||||
|
"role": "role"
|
||||||
|
},
|
||||||
|
"path": "chanceQZhang/zhihuhighvotes",
|
||||||
|
"split": "train",
|
||||||
|
"trust_remote_code": false,
|
||||||
|
"type": "chat_template"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"ddp": false,
|
||||||
|
"device": "cuda:0",
|
||||||
|
"device_map": "auto",
|
||||||
|
"dion_rank_fraction": 1.0,
|
||||||
|
"dion_rank_multiple_of": 1,
|
||||||
|
"env_capabilities": {
|
||||||
|
"torch_version": "2.8.0"
|
||||||
|
},
|
||||||
|
"eval_batch_size": 6,
|
||||||
|
"eval_causal_lm_metrics": [
|
||||||
|
"sacrebleu",
|
||||||
|
"comet",
|
||||||
|
"ter",
|
||||||
|
"chrf"
|
||||||
|
],
|
||||||
|
"eval_max_new_tokens": 128,
|
||||||
|
"eval_sample_packing": true,
|
||||||
|
"eval_steps": 100,
|
||||||
|
"eval_table_size": 0,
|
||||||
|
"experimental_skip_move_to_device": true,
|
||||||
|
"flash_attention": false,
|
||||||
|
"fp16": false,
|
||||||
|
"gradient_accumulation_steps": 3,
|
||||||
|
"gradient_checkpointing": true,
|
||||||
|
"gradient_checkpointing_kwargs": {
|
||||||
|
"use_reentrant": true
|
||||||
|
},
|
||||||
|
"include_tkps": true,
|
||||||
|
"is_falcon_derived_model": false,
|
||||||
|
"is_llama_derived_model": false,
|
||||||
|
"is_mistral_derived_model": false,
|
||||||
|
"learning_rate": 5e-05,
|
||||||
|
"lisa_layers_attribute": "model.layers",
|
||||||
|
"load_best_model_at_end": false,
|
||||||
|
"load_in_4bit": false,
|
||||||
|
"load_in_8bit": false,
|
||||||
|
"local_rank": 0,
|
||||||
|
"logging_steps": 10,
|
||||||
|
"lora_alpha": 32,
|
||||||
|
"lora_dropout": 0.1,
|
||||||
|
"lora_model_dir": "./outputs/zhihu-tech-career-lora",
|
||||||
|
"lora_r": 8,
|
||||||
|
"lora_target_modules": [
|
||||||
|
"q_proj",
|
||||||
|
"v_proj",
|
||||||
|
"k_proj",
|
||||||
|
"o_proj",
|
||||||
|
"gate_proj",
|
||||||
|
"up_proj",
|
||||||
|
"down_proj"
|
||||||
|
],
|
||||||
|
"loraplus_lr_embedding": 1e-06,
|
||||||
|
"lr_scheduler": "cosine",
|
||||||
|
"mean_resizing_embeddings": false,
|
||||||
|
"merge_lora": true,
|
||||||
|
"micro_batch_size": 6,
|
||||||
|
"model_config_type": "qwen2",
|
||||||
|
"num_epochs": 2.0,
|
||||||
|
"optimizer": "adamw_torch_fused",
|
||||||
|
"otel_metrics_host": "localhost",
|
||||||
|
"otel_metrics_port": 8000,
|
||||||
|
"output_dir": "./outputs/zhihu-tech-career-lora",
|
||||||
|
"pad_to_sequence_len": true,
|
||||||
|
"pretrain_multipack_attn": true,
|
||||||
|
"profiler_steps_start": 0,
|
||||||
|
"qlora_sharded_model_loading": false,
|
||||||
|
"ray_num_workers": 1,
|
||||||
|
"resources_per_worker": {
|
||||||
|
"GPU": 1
|
||||||
|
},
|
||||||
|
"sample_packing": true,
|
||||||
|
"sample_packing_bin_size": 200,
|
||||||
|
"sample_packing_group_size": 100000,
|
||||||
|
"save_only_model": false,
|
||||||
|
"save_safetensors": true,
|
||||||
|
"save_steps": 302,
|
||||||
|
"sequence_len": 2048,
|
||||||
|
"shuffle_before_merging_datasets": false,
|
||||||
|
"shuffle_merged_datasets": true,
|
||||||
|
"skip_prepare_dataset": false,
|
||||||
|
"streaming_multipack_buffer_size": 10000,
|
||||||
|
"strict": false,
|
||||||
|
"tensor_parallel_size": 1,
|
||||||
|
"tiled_mlp_use_original_mlp": true,
|
||||||
|
"tokenizer_config": "Orion-zhen/Meissa-Qwen2.5-7B-Instruct",
|
||||||
|
"tokenizer_save_jinja_files": true,
|
||||||
|
"tokenizer_type": "AutoTokenizer",
|
||||||
|
"torch_dtype": "torch.bfloat16",
|
||||||
|
"train_on_inputs": false,
|
||||||
|
"trl": {
|
||||||
|
"log_completions": false,
|
||||||
|
"mask_truncated_completions": false,
|
||||||
|
"ref_model_mixup_alpha": 0.9,
|
||||||
|
"ref_model_sync_steps": 64,
|
||||||
|
"scale_rewards": true,
|
||||||
|
"sync_ref_model": false,
|
||||||
|
"use_vllm": false,
|
||||||
|
"vllm_server_host": "0.0.0.0",
|
||||||
|
"vllm_server_port": 8000
|
||||||
|
},
|
||||||
|
"type_of_model": "AutoModelForCausalLM",
|
||||||
|
"use_otel_metrics": false,
|
||||||
|
"use_ray": false,
|
||||||
|
"val_set_size": 0.0,
|
||||||
|
"vllm": {
|
||||||
|
"device": "auto",
|
||||||
|
"dtype": "auto",
|
||||||
|
"gpu_memory_utilization": 0.9,
|
||||||
|
"host": "0.0.0.0",
|
||||||
|
"port": 8000
|
||||||
|
},
|
||||||
|
"weight_decay": 0.0,
|
||||||
|
"world_size": 1
|
||||||
|
}
|
||||||
|
[2026-01-13 12:24:55,938] [INFO] [axolotl.cli.utils.load.load_model_and_tokenizer:40] [PID:4815] loading tokenizer... Orion-zhen/Meissa-Qwen2.5-7B-Instruct
|
||||||
|
[2026-01-13 12:24:56,619] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:4815] EOS: 151645 / <|im_end|>
|
||||||
|
[2026-01-13 12:24:56,619] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:4815] BOS: None / None
|
||||||
|
[2026-01-13 12:24:56,619] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:4815] PAD: 151643 / <|endoftext|>
|
||||||
|
[2026-01-13 12:24:56,619] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:4815] UNK: None / None
|
||||||
|
[2026-01-13 12:24:56,619] [INFO] [axolotl.cli.utils.load.load_model_and_tokenizer:43] [PID:4815] loading model...
|
||||||
|
[2026-01-13 12:24:56,739] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:4815] Patched Trainer.evaluation_loop with nanmean loss calculation
|
||||||
|
[2026-01-13 12:24:56,740] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:4815] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
|
||||||
|
[2026-01-13 12:24:56,741] [INFO] [axolotl.loaders.patch_manager._apply_multipack_patches:344] [PID:4815] Applying multipack dataloader patch for sample packing...
|
||||||
|
|
||||||
|
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
|
||||||
|
Loading checkpoint shards: 25%|███████████████████████▎ | 1/4 [00:00<00:02, 1.21it/s]
|
||||||
|
Loading checkpoint shards: 50%|██████████████████████████████████████████████▌ | 2/4 [00:01<00:01, 1.21it/s]
|
||||||
|
Loading checkpoint shards: 75%|█████████████████████████████████████████████████████████████████████▊ | 3/4 [00:02<00:00, 1.29it/s]
|
||||||
|
Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:02<00:00, 1.85it/s]
|
||||||
|
Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:02<00:00, 1.57it/s]
|
||||||
24
merged/added_tokens.json
Normal file
24
merged/added_tokens.json
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
{
|
||||||
|
"</tool_call>": 151658,
|
||||||
|
"<tool_call>": 151657,
|
||||||
|
"<|box_end|>": 151649,
|
||||||
|
"<|box_start|>": 151648,
|
||||||
|
"<|endoftext|>": 151643,
|
||||||
|
"<|file_sep|>": 151664,
|
||||||
|
"<|fim_middle|>": 151660,
|
||||||
|
"<|fim_pad|>": 151662,
|
||||||
|
"<|fim_prefix|>": 151659,
|
||||||
|
"<|fim_suffix|>": 151661,
|
||||||
|
"<|im_end|>": 151645,
|
||||||
|
"<|im_start|>": 151644,
|
||||||
|
"<|image_pad|>": 151655,
|
||||||
|
"<|object_ref_end|>": 151647,
|
||||||
|
"<|object_ref_start|>": 151646,
|
||||||
|
"<|quad_end|>": 151651,
|
||||||
|
"<|quad_start|>": 151650,
|
||||||
|
"<|repo_name|>": 151663,
|
||||||
|
"<|video_pad|>": 151656,
|
||||||
|
"<|vision_end|>": 151653,
|
||||||
|
"<|vision_pad|>": 151654,
|
||||||
|
"<|vision_start|>": 151652
|
||||||
|
}
|
||||||
7
merged/chat_template.jinja
Normal file
7
merged/chat_template.jinja
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system
|
||||||
|
' + system_message + '<|im_end|>
|
||||||
|
' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user
|
||||||
|
' + content + '<|im_end|>
|
||||||
|
<|im_start|>assistant
|
||||||
|
' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '
|
||||||
|
' }}{% endif %}{% endfor %}
|
||||||
57
merged/config.json
Normal file
57
merged/config.json
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"Qwen2ForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"dtype": "bfloat16",
|
||||||
|
"eos_token_id": 151645,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 3584,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 18944,
|
||||||
|
"layer_types": [
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention",
|
||||||
|
"full_attention"
|
||||||
|
],
|
||||||
|
"max_position_embeddings": 32768,
|
||||||
|
"max_window_layers": 28,
|
||||||
|
"model_type": "qwen2",
|
||||||
|
"num_attention_heads": 28,
|
||||||
|
"num_hidden_layers": 28,
|
||||||
|
"num_key_value_heads": 4,
|
||||||
|
"rms_norm_eps": 1e-06,
|
||||||
|
"rope_scaling": null,
|
||||||
|
"rope_theta": 1000000.0,
|
||||||
|
"sliding_window": null,
|
||||||
|
"tie_word_embeddings": false,
|
||||||
|
"transformers_version": "4.57.1",
|
||||||
|
"use_cache": true,
|
||||||
|
"use_sliding_window": false,
|
||||||
|
"vocab_size": 152064
|
||||||
|
}
|
||||||
14
merged/generation_config.json
Normal file
14
merged/generation_config.json
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"bos_token_id": 151643,
|
||||||
|
"do_sample": true,
|
||||||
|
"eos_token_id": [
|
||||||
|
151645,
|
||||||
|
151643
|
||||||
|
],
|
||||||
|
"pad_token_id": 151643,
|
||||||
|
"repetition_penalty": 1.05,
|
||||||
|
"temperature": 0.7,
|
||||||
|
"top_k": 20,
|
||||||
|
"top_p": 0.8,
|
||||||
|
"transformers_version": "4.57.1"
|
||||||
|
}
|
||||||
151388
merged/merges.txt
Normal file
151388
merged/merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
merged/model-00001-of-00004.safetensors
Normal file
3
merged/model-00001-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a8fa7b6cf384af456187d8f56bf8732d210c8e6193195e267228f10e52f45378
|
||||||
|
size 4877660776
|
||||||
3
merged/model-00002-of-00004.safetensors
Normal file
3
merged/model-00002-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:69c126e4fd4dec674b105848a668c1791b56a4fedc4c1990c6f5d99128c7027f
|
||||||
|
size 4932751008
|
||||||
3
merged/model-00003-of-00004.safetensors
Normal file
3
merged/model-00003-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:5a5702b4c57ab07595b44426a6de23c76e545a50b288c837a57870728b69ea9e
|
||||||
|
size 4330865200
|
||||||
3
merged/model-00004-of-00004.safetensors
Normal file
3
merged/model-00004-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:06006972c3be88e8a44fe21cfe2b0472b130780c781a741f8f90f1fe5ba3aae2
|
||||||
|
size 1089994880
|
||||||
347
merged/model.safetensors.index.json
Normal file
347
merged/model.safetensors.index.json
Normal file
@@ -0,0 +1,347 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"total_parameters": 7615616512,
|
||||||
|
"total_size": 15231233024
|
||||||
|
},
|
||||||
|
"weight_map": {
|
||||||
|
"lm_head.weight": "model-00004-of-00004.safetensors",
|
||||||
|
"model.embed_tokens.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||||
|
"model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||||
|
"model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
|
||||||
|
"model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||||
|
"model.norm.weight": "model-00003-of-00004.safetensors"
|
||||||
|
}
|
||||||
|
}
|
||||||
31
merged/special_tokens_map.json
Normal file
31
merged/special_tokens_map.json
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
{
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<|object_ref_start|>",
|
||||||
|
"<|object_ref_end|>",
|
||||||
|
"<|box_start|>",
|
||||||
|
"<|box_end|>",
|
||||||
|
"<|quad_start|>",
|
||||||
|
"<|quad_end|>",
|
||||||
|
"<|vision_start|>",
|
||||||
|
"<|vision_end|>",
|
||||||
|
"<|vision_pad|>",
|
||||||
|
"<|image_pad|>",
|
||||||
|
"<|video_pad|>"
|
||||||
|
],
|
||||||
|
"eos_token": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"pad_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
}
|
||||||
|
}
|
||||||
3
merged/tokenizer.json
Normal file
3
merged/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
|
||||||
|
size 11421896
|
||||||
208
merged/tokenizer_config.json
Normal file
208
merged/tokenizer_config.json
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
{
|
||||||
|
"add_bos_token": false,
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"added_tokens_decoder": {
|
||||||
|
"151643": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151644": {
|
||||||
|
"content": "<|im_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151645": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151646": {
|
||||||
|
"content": "<|object_ref_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151647": {
|
||||||
|
"content": "<|object_ref_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151648": {
|
||||||
|
"content": "<|box_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151649": {
|
||||||
|
"content": "<|box_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151650": {
|
||||||
|
"content": "<|quad_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151651": {
|
||||||
|
"content": "<|quad_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151652": {
|
||||||
|
"content": "<|vision_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151653": {
|
||||||
|
"content": "<|vision_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151654": {
|
||||||
|
"content": "<|vision_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151655": {
|
||||||
|
"content": "<|image_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151656": {
|
||||||
|
"content": "<|video_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151657": {
|
||||||
|
"content": "<tool_call>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151658": {
|
||||||
|
"content": "</tool_call>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151659": {
|
||||||
|
"content": "<|fim_prefix|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151660": {
|
||||||
|
"content": "<|fim_middle|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151661": {
|
||||||
|
"content": "<|fim_suffix|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151662": {
|
||||||
|
"content": "<|fim_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151663": {
|
||||||
|
"content": "<|repo_name|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151664": {
|
||||||
|
"content": "<|file_sep|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<|object_ref_start|>",
|
||||||
|
"<|object_ref_end|>",
|
||||||
|
"<|box_start|>",
|
||||||
|
"<|box_end|>",
|
||||||
|
"<|quad_start|>",
|
||||||
|
"<|quad_end|>",
|
||||||
|
"<|vision_start|>",
|
||||||
|
"<|vision_end|>",
|
||||||
|
"<|vision_pad|>",
|
||||||
|
"<|image_pad|>",
|
||||||
|
"<|video_pad|>"
|
||||||
|
],
|
||||||
|
"bos_token": null,
|
||||||
|
"clean_up_tokenization_spaces": false,
|
||||||
|
"eos_token": "<|im_end|>",
|
||||||
|
"errors": "replace",
|
||||||
|
"extra_special_tokens": {},
|
||||||
|
"model_max_length": 131072,
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"padding_side": "left",
|
||||||
|
"split_special_tokens": false,
|
||||||
|
"tokenizer_class": "Qwen2Tokenizer",
|
||||||
|
"unk_token": null
|
||||||
|
}
|
||||||
1
merged/vocab.json
Normal file
1
merged/vocab.json
Normal file
File diff suppressed because one or more lines are too long
3
merged/zhihu_f16.gguf
Normal file
3
merged/zhihu_f16.gguf
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:38c500885a4f00d1f88d04b3568ce63d726928ad3a82b65ad3cd31e693cefcec
|
||||||
|
size 15237851168
|
||||||
3
merged/zhihu_model_q5_km.gguf
Normal file
3
merged/zhihu_model_q5_km.gguf
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:746585e58df0df5ed609e5a1eb02a4ed9da899a2e4f65c5018e212a87e0387ff
|
||||||
|
size 5444829216
|
||||||
151388
merges.txt
Normal file
151388
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
31
special_tokens_map.json
Normal file
31
special_tokens_map.json
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
{
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<|object_ref_start|>",
|
||||||
|
"<|object_ref_end|>",
|
||||||
|
"<|box_start|>",
|
||||||
|
"<|box_end|>",
|
||||||
|
"<|quad_start|>",
|
||||||
|
"<|quad_end|>",
|
||||||
|
"<|vision_start|>",
|
||||||
|
"<|vision_end|>",
|
||||||
|
"<|vision_pad|>",
|
||||||
|
"<|image_pad|>",
|
||||||
|
"<|video_pad|>"
|
||||||
|
],
|
||||||
|
"eos_token": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"pad_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
}
|
||||||
|
}
|
||||||
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
|
||||||
|
size 11421896
|
||||||
208
tokenizer_config.json
Normal file
208
tokenizer_config.json
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
{
|
||||||
|
"add_bos_token": false,
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"added_tokens_decoder": {
|
||||||
|
"151643": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151644": {
|
||||||
|
"content": "<|im_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151645": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151646": {
|
||||||
|
"content": "<|object_ref_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151647": {
|
||||||
|
"content": "<|object_ref_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151648": {
|
||||||
|
"content": "<|box_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151649": {
|
||||||
|
"content": "<|box_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151650": {
|
||||||
|
"content": "<|quad_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151651": {
|
||||||
|
"content": "<|quad_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151652": {
|
||||||
|
"content": "<|vision_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151653": {
|
||||||
|
"content": "<|vision_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151654": {
|
||||||
|
"content": "<|vision_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151655": {
|
||||||
|
"content": "<|image_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151656": {
|
||||||
|
"content": "<|video_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"151657": {
|
||||||
|
"content": "<tool_call>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151658": {
|
||||||
|
"content": "</tool_call>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151659": {
|
||||||
|
"content": "<|fim_prefix|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151660": {
|
||||||
|
"content": "<|fim_middle|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151661": {
|
||||||
|
"content": "<|fim_suffix|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151662": {
|
||||||
|
"content": "<|fim_pad|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151663": {
|
||||||
|
"content": "<|repo_name|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
},
|
||||||
|
"151664": {
|
||||||
|
"content": "<|file_sep|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<|object_ref_start|>",
|
||||||
|
"<|object_ref_end|>",
|
||||||
|
"<|box_start|>",
|
||||||
|
"<|box_end|>",
|
||||||
|
"<|quad_start|>",
|
||||||
|
"<|quad_end|>",
|
||||||
|
"<|vision_start|>",
|
||||||
|
"<|vision_end|>",
|
||||||
|
"<|vision_pad|>",
|
||||||
|
"<|image_pad|>",
|
||||||
|
"<|video_pad|>"
|
||||||
|
],
|
||||||
|
"bos_token": null,
|
||||||
|
"clean_up_tokenization_spaces": false,
|
||||||
|
"eos_token": "<|im_end|>",
|
||||||
|
"errors": "replace",
|
||||||
|
"extra_special_tokens": {},
|
||||||
|
"model_max_length": 131072,
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"padding_side": "left",
|
||||||
|
"split_special_tokens": false,
|
||||||
|
"tokenizer_class": "Qwen2Tokenizer",
|
||||||
|
"unk_token": null
|
||||||
|
}
|
||||||
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user