初始化项目，由ModelHub XC社区提供模型
Model: dilkushsingh/qwen2.5-0.5b-nlp2bash-FFT Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
58
README.md
Normal file
58
README.md
Normal file
@@ -0,0 +1,58 @@
|
||||
---
|
||||
base_model: Qwen/Qwen2.5-0.5B-Instruct
|
||||
library_name: transformers
|
||||
model_name: FFT
|
||||
tags:
|
||||
- generated_from_trainer
|
||||
- sft
|
||||
- trl
|
||||
licence: license
|
||||
---
|
||||
|
||||
# Model Card for FFT
|
||||
|
||||
This model is a fine-tuned version of [Qwen/Qwen2.5-0.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct).
|
||||
It has been trained using [TRL](https://github.com/huggingface/trl).
|
||||
|
||||
## Quick start
|
||||
|
||||
```python
|
||||
from transformers import pipeline
|
||||
|
||||
question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
|
||||
generator = pipeline("text-generation", model="None", device="cuda")
|
||||
output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
|
||||
print(output["generated_text"])
|
||||
```
|
||||
|
||||
## Training procedure
|
||||
|
||||
[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/dilkushsingh-mu/nlp2bash-finetuning/runs/pu4u1vuf)
|
||||
|
||||
|
||||
|
||||
This model was trained with SFT.
|
||||
|
||||
### Framework versions
|
||||
|
||||
- TRL: 1.7.0
|
||||
- Transformers: 5.12.1
|
||||
- Pytorch: 2.10.0+cu128
|
||||
- Datasets: 5.0.0
|
||||
- Tokenizers: 0.22.2
|
||||
|
||||
## Citations
|
||||
|
||||
|
||||
|
||||
Cite TRL as:
|
||||
|
||||
```bibtex
|
||||
@software{vonwerra2020trl,
|
||||
title = {{TRL: Transformers Reinforcement Learning}},
|
||||
author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
|
||||
license = {Apache-2.0},
|
||||
url = {https://github.com/huggingface/trl},
|
||||
year = {2020}
|
||||
}
|
||||
```
|
||||
54
chat_template.jinja
Normal file
54
chat_template.jinja
Normal file
@@ -0,0 +1,54 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- messages[0]['content'] }}
|
||||
{%- else %}
|
||||
{{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
|
||||
{%- endif %}
|
||||
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role }}
|
||||
{%- if message.content %}
|
||||
{{- '\n' + message.content }}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
54
checkpoint-1126/chat_template.jinja
Normal file
54
checkpoint-1126/chat_template.jinja
Normal file
@@ -0,0 +1,54 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- messages[0]['content'] }}
|
||||
{%- else %}
|
||||
{{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
|
||||
{%- endif %}
|
||||
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role }}
|
||||
{%- if message.content %}
|
||||
{{- '\n' + message.content }}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
57
checkpoint-1126/config.json
Normal file
57
checkpoint-1126/config.json
Normal file
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen2ForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": null,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 151645,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 896,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 4864,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 21,
|
||||
"model_type": "qwen2",
|
||||
"num_attention_heads": 14,
|
||||
"num_hidden_layers": 24,
|
||||
"num_key_value_heads": 2,
|
||||
"pad_token_id": 151643,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.12.1",
|
||||
"use_cache": false,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
13
checkpoint-1126/generation_config.json
Normal file
13
checkpoint-1126/generation_config.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"repetition_penalty": 1.1,
|
||||
"temperature": 0.7,
|
||||
"top_k": 20,
|
||||
"top_p": 0.8,
|
||||
"transformers_version": "5.12.1"
|
||||
}
|
||||
3
checkpoint-1126/model.safetensors
Normal file
3
checkpoint-1126/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:b5f8b69d19a2f71c729d2eb592d3ed38ab299591b0ac2f2a2620f21d1a501c74
|
||||
size 988097824
|
||||
3
checkpoint-1126/optimizer.pt
Normal file
3
checkpoint-1126/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:66fc67f0a0392f7788538ccc2780ae366437c3601a0f03cc755ba492eb46bfb0
|
||||
size 1976378699
|
||||
3
checkpoint-1126/rng_state.pth
Normal file
3
checkpoint-1126/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:56aae6a7cc55179a5619c69bb9f81c5ad57b6d62a6d5e344f6623030c0d95b02
|
||||
size 14645
|
||||
3
checkpoint-1126/scheduler.pt
Normal file
3
checkpoint-1126/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f92c37453adcfaf88f0707474d73368680d6cbff21a50d73f67ca3980edf1765
|
||||
size 1465
|
||||
3
checkpoint-1126/tokenizer.json
Normal file
3
checkpoint-1126/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
|
||||
size 11421892
|
||||
30
checkpoint-1126/tokenizer_config.json
Normal file
30
checkpoint-1126/tokenizer_config.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"backend": "tokenizers",
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"is_local": false,
|
||||
"local_files_only": false,
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
506
checkpoint-1126/trainer_state.json
Normal file
506
checkpoint-1126/trainer_state.json
Normal file
@@ -0,0 +1,506 @@
|
||||
{
|
||||
"best_global_step": 1126,
|
||||
"best_metric": 1.2738397121429443,
|
||||
"best_model_checkpoint": "finetuned_models/FFT/checkpoint-1126",
|
||||
"epoch": 2.0,
|
||||
"eval_steps": 500,
|
||||
"global_step": 1126,
|
||||
"is_hyper_param_search": false,
|
||||
"is_local_process_zero": true,
|
||||
"is_world_process_zero": true,
|
||||
"log_history": [
|
||||
{
|
||||
"entropy": 1.5691047963500022,
|
||||
"epoch": 0.044444444444444446,
|
||||
"grad_norm": 61.75,
|
||||
"learning_rate": 2.840236686390533e-06,
|
||||
"loss": 2.82507080078125,
|
||||
"mean_token_accuracy": 0.5578909605741501,
|
||||
"num_tokens": 43581.0,
|
||||
"step": 25
|
||||
},
|
||||
{
|
||||
"entropy": 1.912336548268795,
|
||||
"epoch": 0.08888888888888889,
|
||||
"grad_norm": 41.25,
|
||||
"learning_rate": 5.7988165680473375e-06,
|
||||
"loss": 2.3563003540039062,
|
||||
"mean_token_accuracy": 0.5931275233626365,
|
||||
"num_tokens": 86889.0,
|
||||
"step": 50
|
||||
},
|
||||
{
|
||||
"entropy": 2.1532553315162657,
|
||||
"epoch": 0.13333333333333333,
|
||||
"grad_norm": 31.125,
|
||||
"learning_rate": 8.757396449704143e-06,
|
||||
"loss": 2.159627685546875,
|
||||
"mean_token_accuracy": 0.5959913417696953,
|
||||
"num_tokens": 130613.0,
|
||||
"step": 75
|
||||
},
|
||||
{
|
||||
"entropy": 1.9658770501613616,
|
||||
"epoch": 0.17777777777777778,
|
||||
"grad_norm": 30.0,
|
||||
"learning_rate": 1.1715976331360948e-05,
|
||||
"loss": 1.9642219543457031,
|
||||
"mean_token_accuracy": 0.6274332204461097,
|
||||
"num_tokens": 174274.0,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"entropy": 1.9181486976146698,
|
||||
"epoch": 0.2222222222222222,
|
||||
"grad_norm": 23.875,
|
||||
"learning_rate": 1.4674556213017752e-05,
|
||||
"loss": 1.9006834411621094,
|
||||
"mean_token_accuracy": 0.6362838646769524,
|
||||
"num_tokens": 217619.0,
|
||||
"step": 125
|
||||
},
|
||||
{
|
||||
"entropy": 1.8227408236265183,
|
||||
"epoch": 0.26666666666666666,
|
||||
"grad_norm": 29.5,
|
||||
"learning_rate": 1.7633136094674557e-05,
|
||||
"loss": 1.834572296142578,
|
||||
"mean_token_accuracy": 0.645097343325615,
|
||||
"num_tokens": 261407.0,
|
||||
"step": 150
|
||||
},
|
||||
{
|
||||
"entropy": 1.8350859326124191,
|
||||
"epoch": 0.3111111111111111,
|
||||
"grad_norm": 25.625,
|
||||
"learning_rate": 1.999946602771351e-05,
|
||||
"loss": 1.8147303771972656,
|
||||
"mean_token_accuracy": 0.6461810152232647,
|
||||
"num_tokens": 305402.0,
|
||||
"step": 175
|
||||
},
|
||||
{
|
||||
"entropy": 1.753722729086876,
|
||||
"epoch": 0.35555555555555557,
|
||||
"grad_norm": 31.5,
|
||||
"learning_rate": 1.9980782984658682e-05,
|
||||
"loss": 1.758114013671875,
|
||||
"mean_token_accuracy": 0.653534509241581,
|
||||
"num_tokens": 348998.0,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"entropy": 1.7685393619537353,
|
||||
"epoch": 0.4,
|
||||
"grad_norm": 23.75,
|
||||
"learning_rate": 1.9935458325191365e-05,
|
||||
"loss": 1.7483135986328124,
|
||||
"mean_token_accuracy": 0.6603708426654339,
|
||||
"num_tokens": 392837.0,
|
||||
"step": 225
|
||||
},
|
||||
{
|
||||
"entropy": 1.7378248673677446,
|
||||
"epoch": 0.4444444444444444,
|
||||
"grad_norm": 26.5,
|
||||
"learning_rate": 1.9863613034027224e-05,
|
||||
"loss": 1.7749085998535157,
|
||||
"mean_token_accuracy": 0.65810049533844,
|
||||
"num_tokens": 436281.0,
|
||||
"step": 250
|
||||
},
|
||||
{
|
||||
"entropy": 1.6766968676447869,
|
||||
"epoch": 0.4888888888888889,
|
||||
"grad_norm": 23.25,
|
||||
"learning_rate": 1.9765438887171327e-05,
|
||||
"loss": 1.668856658935547,
|
||||
"mean_token_accuracy": 0.6644712510704994,
|
||||
"num_tokens": 480451.0,
|
||||
"step": 275
|
||||
},
|
||||
{
|
||||
"entropy": 1.669287955760956,
|
||||
"epoch": 0.5333333333333333,
|
||||
"grad_norm": 20.625,
|
||||
"learning_rate": 1.9641197940012136e-05,
|
||||
"loss": 1.648075714111328,
|
||||
"mean_token_accuracy": 0.6689342276751995,
|
||||
"num_tokens": 524418.0,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"entropy": 1.587462457716465,
|
||||
"epoch": 0.5777777777777777,
|
||||
"grad_norm": 21.625,
|
||||
"learning_rate": 1.9491221827819348e-05,
|
||||
"loss": 1.5674880981445312,
|
||||
"mean_token_accuracy": 0.6790469540655613,
|
||||
"num_tokens": 568093.0,
|
||||
"step": 325
|
||||
},
|
||||
{
|
||||
"entropy": 1.5877643951773643,
|
||||
"epoch": 0.6222222222222222,
|
||||
"grad_norm": 23.5,
|
||||
"learning_rate": 1.9315910880512792e-05,
|
||||
"loss": 1.5833602905273438,
|
||||
"mean_token_accuracy": 0.67303916528821,
|
||||
"num_tokens": 611875.0,
|
||||
"step": 350
|
||||
},
|
||||
{
|
||||
"entropy": 1.5628123182058333,
|
||||
"epoch": 0.6666666666666666,
|
||||
"grad_norm": 19.875,
|
||||
"learning_rate": 1.911573305406528e-05,
|
||||
"loss": 1.5389964294433593,
|
||||
"mean_token_accuracy": 0.6916262510418892,
|
||||
"num_tokens": 655258.0,
|
||||
"step": 375
|
||||
},
|
||||
{
|
||||
"entropy": 1.5605447962880135,
|
||||
"epoch": 0.7111111111111111,
|
||||
"grad_norm": 25.25,
|
||||
"learning_rate": 1.8891222681391853e-05,
|
||||
"loss": 1.5674354553222656,
|
||||
"mean_token_accuracy": 0.6811535279452801,
|
||||
"num_tokens": 698915.0,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"entropy": 1.5302811539173127,
|
||||
"epoch": 0.7555555555555555,
|
||||
"grad_norm": 21.0,
|
||||
"learning_rate": 1.8642979046059595e-05,
|
||||
"loss": 1.5149029541015624,
|
||||
"mean_token_accuracy": 0.6928434443473815,
|
||||
"num_tokens": 742608.0,
|
||||
"step": 425
|
||||
},
|
||||
{
|
||||
"entropy": 1.4984435787796975,
|
||||
"epoch": 0.8,
|
||||
"grad_norm": 20.5,
|
||||
"learning_rate": 1.8371664782625287e-05,
|
||||
"loss": 1.4783544921875,
|
||||
"mean_token_accuracy": 0.6922086542844772,
|
||||
"num_tokens": 785924.0,
|
||||
"step": 450
|
||||
},
|
||||
{
|
||||
"entropy": 1.4762829530239105,
|
||||
"epoch": 0.8444444444444444,
|
||||
"grad_norm": 22.875,
|
||||
"learning_rate": 1.8078004107870797e-05,
|
||||
"loss": 1.4516114807128906,
|
||||
"mean_token_accuracy": 0.702991289794445,
|
||||
"num_tokens": 829069.0,
|
||||
"step": 475
|
||||
},
|
||||
{
|
||||
"entropy": 1.5141191801428795,
|
||||
"epoch": 0.8888888888888888,
|
||||
"grad_norm": 20.75,
|
||||
"learning_rate": 1.7762780887657576e-05,
|
||||
"loss": 1.5113209533691405,
|
||||
"mean_token_accuracy": 0.6913984973728656,
|
||||
"num_tokens": 872824.0,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"entropy": 1.4147853454947472,
|
||||
"epoch": 0.9333333333333333,
|
||||
"grad_norm": 25.75,
|
||||
"learning_rate": 1.74268365445604e-05,
|
||||
"loss": 1.394475555419922,
|
||||
"mean_token_accuracy": 0.712423055768013,
|
||||
"num_tokens": 916212.0,
|
||||
"step": 525
|
||||
},
|
||||
{
|
||||
"entropy": 1.4362201416492462,
|
||||
"epoch": 0.9777777777777777,
|
||||
"grad_norm": 21.25,
|
||||
"learning_rate": 1.7071067811865477e-05,
|
||||
"loss": 1.4004106140136718,
|
||||
"mean_token_accuracy": 0.7106147593259812,
|
||||
"num_tokens": 959695.0,
|
||||
"step": 550
|
||||
},
|
||||
{
|
||||
"epoch": 1.0,
|
||||
"eval_entropy": 1.3886345291137696,
|
||||
"eval_loss": 1.3774118423461914,
|
||||
"eval_mean_token_accuracy": 0.7105021185874939,
|
||||
"eval_num_tokens": 981504.0,
|
||||
"eval_runtime": 28.7124,
|
||||
"eval_samples_per_second": 34.828,
|
||||
"eval_steps_per_second": 4.354,
|
||||
"step": 563
|
||||
},
|
||||
{
|
||||
"entropy": 1.3702899970266285,
|
||||
"epoch": 1.0213333333333334,
|
||||
"grad_norm": 24.375,
|
||||
"learning_rate": 1.6696424339928153e-05,
|
||||
"loss": 1.2998457336425782,
|
||||
"mean_token_accuracy": 0.7251553581077226,
|
||||
"num_tokens": 1002442.0,
|
||||
"step": 575
|
||||
},
|
||||
{
|
||||
"entropy": 1.2198174016177654,
|
||||
"epoch": 1.0657777777777777,
|
||||
"grad_norm": 19.0,
|
||||
"learning_rate": 1.6303906161279554e-05,
|
||||
"loss": 1.1578231048583985,
|
||||
"mean_token_accuracy": 0.7492367821931839,
|
||||
"num_tokens": 1045974.0,
|
||||
"step": 600
|
||||
},
|
||||
{
|
||||
"entropy": 1.2214804862439632,
|
||||
"epoch": 1.1102222222222222,
|
||||
"grad_norm": 21.5,
|
||||
"learning_rate": 1.5894561021248535e-05,
|
||||
"loss": 1.1222823333740235,
|
||||
"mean_token_accuracy": 0.7596075300872326,
|
||||
"num_tokens": 1089691.0,
|
||||
"step": 625
|
||||
},
|
||||
{
|
||||
"entropy": 1.230571178793907,
|
||||
"epoch": 1.1546666666666667,
|
||||
"grad_norm": 18.25,
|
||||
"learning_rate": 1.5469481581224274e-05,
|
||||
"loss": 1.1252061462402343,
|
||||
"mean_token_accuracy": 0.7554993903636933,
|
||||
"num_tokens": 1133544.0,
|
||||
"step": 650
|
||||
},
|
||||
{
|
||||
"entropy": 1.2035815131664276,
|
||||
"epoch": 1.199111111111111,
|
||||
"grad_norm": 17.75,
|
||||
"learning_rate": 1.5029802502024788e-05,
|
||||
"loss": 1.1525257110595704,
|
||||
"mean_token_accuracy": 0.7539819921553135,
|
||||
"num_tokens": 1176944.0,
|
||||
"step": 675
|
||||
},
|
||||
{
|
||||
"entropy": 1.1947351975739002,
|
||||
"epoch": 1.2435555555555555,
|
||||
"grad_norm": 19.5,
|
||||
"learning_rate": 1.4576697415156818e-05,
|
||||
"loss": 1.1312750244140626,
|
||||
"mean_token_accuracy": 0.755437722504139,
|
||||
"num_tokens": 1220536.0,
|
||||
"step": 700
|
||||
},
|
||||
{
|
||||
"entropy": 1.2491805362701416,
|
||||
"epoch": 1.288,
|
||||
"grad_norm": 18.625,
|
||||
"learning_rate": 1.4111375790051511e-05,
|
||||
"loss": 1.1570246124267578,
|
||||
"mean_token_accuracy": 0.7577420398592949,
|
||||
"num_tokens": 1264076.0,
|
||||
"step": 725
|
||||
},
|
||||
{
|
||||
"entropy": 1.2654732164740563,
|
||||
"epoch": 1.3324444444444445,
|
||||
"grad_norm": 20.875,
|
||||
"learning_rate": 1.3635079705638298e-05,
|
||||
"loss": 1.1855599212646484,
|
||||
"mean_token_accuracy": 0.751485439389944,
|
||||
"num_tokens": 1308034.0,
|
||||
"step": 750
|
||||
},
|
||||
{
|
||||
"entropy": 1.2152655026316643,
|
||||
"epoch": 1.3768888888888888,
|
||||
"grad_norm": 20.25,
|
||||
"learning_rate": 1.3149080534874519e-05,
|
||||
"loss": 1.1257845306396483,
|
||||
"mean_token_accuracy": 0.7632686151564121,
|
||||
"num_tokens": 1351646.0,
|
||||
"step": 775
|
||||
},
|
||||
{
|
||||
"entropy": 1.193697052896023,
|
||||
"epoch": 1.4213333333333333,
|
||||
"grad_norm": 24.75,
|
||||
"learning_rate": 1.2654675551080724e-05,
|
||||
"loss": 1.121256103515625,
|
||||
"mean_token_accuracy": 0.7563131806254387,
|
||||
"num_tokens": 1395047.0,
|
||||
"step": 800
|
||||
},
|
||||
{
|
||||
"entropy": 1.1570866696536541,
|
||||
"epoch": 1.4657777777777778,
|
||||
"grad_norm": 22.25,
|
||||
"learning_rate": 1.2153184465140413e-05,
|
||||
"loss": 1.0696445465087892,
|
||||
"mean_token_accuracy": 0.765836420059204,
|
||||
"num_tokens": 1438280.0,
|
||||
"step": 825
|
||||
},
|
||||
{
|
||||
"entropy": 1.151665413379669,
|
||||
"epoch": 1.5102222222222221,
|
||||
"grad_norm": 17.75,
|
||||
"learning_rate": 1.164594590280734e-05,
|
||||
"loss": 1.0663511657714844,
|
||||
"mean_token_accuracy": 0.7644271370768547,
|
||||
"num_tokens": 1481978.0,
|
||||
"step": 850
|
||||
},
|
||||
{
|
||||
"entropy": 1.2120365902781487,
|
||||
"epoch": 1.5546666666666666,
|
||||
"grad_norm": 17.5,
|
||||
"learning_rate": 1.1134313831523547e-05,
|
||||
"loss": 1.1270946502685546,
|
||||
"mean_token_accuracy": 0.7585732811689376,
|
||||
"num_tokens": 1525597.0,
|
||||
"step": 875
|
||||
},
|
||||
{
|
||||
"entropy": 1.151113938242197,
|
||||
"epoch": 1.5991111111111111,
|
||||
"grad_norm": 21.5,
|
||||
"learning_rate": 1.0619653946285948e-05,
|
||||
"loss": 1.0662925720214844,
|
||||
"mean_token_accuracy": 0.7686286368966102,
|
||||
"num_tokens": 1568743.0,
|
||||
"step": 900
|
||||
},
|
||||
{
|
||||
"entropy": 1.162338248193264,
|
||||
"epoch": 1.6435555555555554,
|
||||
"grad_norm": 20.625,
|
||||
"learning_rate": 1.0103340024208674e-05,
|
||||
"loss": 1.0983226013183593,
|
||||
"mean_token_accuracy": 0.7634404581785202,
|
||||
"num_tokens": 1612238.0,
|
||||
"step": 925
|
||||
},
|
||||
{
|
||||
"entropy": 1.1956017450988292,
|
||||
"epoch": 1.688,
|
||||
"grad_norm": 18.625,
|
||||
"learning_rate": 9.586750257511868e-06,
|
||||
"loss": 1.1054475402832031,
|
||||
"mean_token_accuracy": 0.7624230819940567,
|
||||
"num_tokens": 1656115.0,
|
||||
"step": 950
|
||||
},
|
||||
{
|
||||
"entropy": 1.1524758899211884,
|
||||
"epoch": 1.7324444444444445,
|
||||
"grad_norm": 20.875,
|
||||
"learning_rate": 9.0712635747253e-06,
|
||||
"loss": 1.063654327392578,
|
||||
"mean_token_accuracy": 0.7672438740730285,
|
||||
"num_tokens": 1700143.0,
|
||||
"step": 975
|
||||
},
|
||||
{
|
||||
"entropy": 1.1726488883793353,
|
||||
"epoch": 1.7768888888888887,
|
||||
"grad_norm": 18.25,
|
||||
"learning_rate": 8.558255959926533e-06,
|
||||
"loss": 1.0800904846191406,
|
||||
"mean_token_accuracy": 0.767799711227417,
|
||||
"num_tokens": 1743486.0,
|
||||
"step": 1000
|
||||
},
|
||||
{
|
||||
"entropy": 1.1595972776412964,
|
||||
"epoch": 1.8213333333333335,
|
||||
"grad_norm": 22.125,
|
||||
"learning_rate": 8.04909677983872e-06,
|
||||
"loss": 1.100532989501953,
|
||||
"mean_token_accuracy": 0.7661877936124801,
|
||||
"num_tokens": 1787577.0,
|
||||
"step": 1025
|
||||
},
|
||||
{
|
||||
"entropy": 1.1869910764694214,
|
||||
"epoch": 1.8657777777777778,
|
||||
"grad_norm": 18.625,
|
||||
"learning_rate": 7.545145128592009e-06,
|
||||
"loss": 1.1068744659423828,
|
||||
"mean_token_accuracy": 0.7658174222707749,
|
||||
"num_tokens": 1831146.0,
|
||||
"step": 1050
|
||||
},
|
||||
{
|
||||
"entropy": 1.1678849336504937,
|
||||
"epoch": 1.9102222222222223,
|
||||
"grad_norm": 20.25,
|
||||
"learning_rate": 7.0477461999055365e-06,
|
||||
"loss": 1.104344940185547,
|
||||
"mean_token_accuracy": 0.7647965340316296,
|
||||
"num_tokens": 1875315.0,
|
||||
"step": 1075
|
||||
},
|
||||
{
|
||||
"entropy": 1.2021603246033192,
|
||||
"epoch": 1.9546666666666668,
|
||||
"grad_norm": 20.75,
|
||||
"learning_rate": 6.558227696373617e-06,
|
||||
"loss": 1.1086157989501952,
|
||||
"mean_token_accuracy": 0.7716675865650177,
|
||||
"num_tokens": 1918868.0,
|
||||
"step": 1100
|
||||
},
|
||||
{
|
||||
"entropy": 1.1465536914765835,
|
||||
"epoch": 1.999111111111111,
|
||||
"grad_norm": 24.0,
|
||||
"learning_rate": 6.077896285440874e-06,
|
||||
"loss": 1.0573983001708984,
|
||||
"mean_token_accuracy": 0.7682515250146389,
|
||||
"num_tokens": 1962187.0,
|
||||
"step": 1125
|
||||
},
|
||||
{
|
||||
"epoch": 2.0,
|
||||
"eval_entropy": 1.1745851793289184,
|
||||
"eval_loss": 1.2738397121429443,
|
||||
"eval_mean_token_accuracy": 0.730101492881775,
|
||||
"eval_num_tokens": 1963008.0,
|
||||
"eval_runtime": 28.609,
|
||||
"eval_samples_per_second": 34.954,
|
||||
"eval_steps_per_second": 4.369,
|
||||
"step": 1126
|
||||
}
|
||||
],
|
||||
"logging_steps": 25,
|
||||
"max_steps": 1689,
|
||||
"num_input_tokens_seen": 0,
|
||||
"num_train_epochs": 3,
|
||||
"save_steps": 500,
|
||||
"stateful_callbacks": {
|
||||
"TrainerControl": {
|
||||
"args": {
|
||||
"should_epoch_stop": false,
|
||||
"should_evaluate": false,
|
||||
"should_log": false,
|
||||
"should_save": true,
|
||||
"should_training_stop": false
|
||||
},
|
||||
"attributes": {}
|
||||
}
|
||||
},
|
||||
"total_flos": 5048150006575104.0,
|
||||
"train_batch_size": 4,
|
||||
"trial_name": null,
|
||||
"trial_params": null
|
||||
}
|
||||
3
checkpoint-1126/training_args.bin
Normal file
3
checkpoint-1126/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:355351641492ac43c5b6480ea76c78a8467059a8b861f06d7c50a5d5d9ef8de5
|
||||
size 5777
|
||||
54
checkpoint-1689/chat_template.jinja
Normal file
54
checkpoint-1689/chat_template.jinja
Normal file
@@ -0,0 +1,54 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- messages[0]['content'] }}
|
||||
{%- else %}
|
||||
{{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
|
||||
{%- endif %}
|
||||
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role }}
|
||||
{%- if message.content %}
|
||||
{{- '\n' + message.content }}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
57
checkpoint-1689/config.json
Normal file
57
checkpoint-1689/config.json
Normal file
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen2ForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": null,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 151645,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 896,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 4864,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 21,
|
||||
"model_type": "qwen2",
|
||||
"num_attention_heads": 14,
|
||||
"num_hidden_layers": 24,
|
||||
"num_key_value_heads": 2,
|
||||
"pad_token_id": 151643,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.12.1",
|
||||
"use_cache": false,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
13
checkpoint-1689/generation_config.json
Normal file
13
checkpoint-1689/generation_config.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"repetition_penalty": 1.1,
|
||||
"temperature": 0.7,
|
||||
"top_k": 20,
|
||||
"top_p": 0.8,
|
||||
"transformers_version": "5.12.1"
|
||||
}
|
||||
3
checkpoint-1689/model.safetensors
Normal file
3
checkpoint-1689/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:78aaf8738cdb4013348217752054bbaca4981d5440cc4905f61e8fa9fa433213
|
||||
size 988097824
|
||||
3
checkpoint-1689/optimizer.pt
Normal file
3
checkpoint-1689/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:996fdf7659149fcff001890ff3325d394ba20316f1760c3960488a6358d997b1
|
||||
size 1976378699
|
||||
3
checkpoint-1689/rng_state.pth
Normal file
3
checkpoint-1689/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:61b5b77e9aceb5804f11a1b38ed2c5391729d8c6966ed20e968822cc25bedd70
|
||||
size 14645
|
||||
3
checkpoint-1689/scheduler.pt
Normal file
3
checkpoint-1689/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:148be883ba879fc6d9643e74f8bb9f6088b9fc21b064b81dc2a6e3d83d8aac3c
|
||||
size 1465
|
||||
3
checkpoint-1689/tokenizer.json
Normal file
3
checkpoint-1689/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
|
||||
size 11421892
|
||||
30
checkpoint-1689/tokenizer_config.json
Normal file
30
checkpoint-1689/tokenizer_config.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"backend": "tokenizers",
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"is_local": false,
|
||||
"local_files_only": false,
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
737
checkpoint-1689/trainer_state.json
Normal file
737
checkpoint-1689/trainer_state.json
Normal file
@@ -0,0 +1,737 @@
|
||||
{
|
||||
"best_global_step": 1126,
|
||||
"best_metric": 1.2738397121429443,
|
||||
"best_model_checkpoint": "finetuned_models/FFT/checkpoint-1126",
|
||||
"epoch": 3.0,
|
||||
"eval_steps": 500,
|
||||
"global_step": 1689,
|
||||
"is_hyper_param_search": false,
|
||||
"is_local_process_zero": true,
|
||||
"is_world_process_zero": true,
|
||||
"log_history": [
|
||||
{
|
||||
"entropy": 1.5691047963500022,
|
||||
"epoch": 0.044444444444444446,
|
||||
"grad_norm": 61.75,
|
||||
"learning_rate": 2.840236686390533e-06,
|
||||
"loss": 2.82507080078125,
|
||||
"mean_token_accuracy": 0.5578909605741501,
|
||||
"num_tokens": 43581.0,
|
||||
"step": 25
|
||||
},
|
||||
{
|
||||
"entropy": 1.912336548268795,
|
||||
"epoch": 0.08888888888888889,
|
||||
"grad_norm": 41.25,
|
||||
"learning_rate": 5.7988165680473375e-06,
|
||||
"loss": 2.3563003540039062,
|
||||
"mean_token_accuracy": 0.5931275233626365,
|
||||
"num_tokens": 86889.0,
|
||||
"step": 50
|
||||
},
|
||||
{
|
||||
"entropy": 2.1532553315162657,
|
||||
"epoch": 0.13333333333333333,
|
||||
"grad_norm": 31.125,
|
||||
"learning_rate": 8.757396449704143e-06,
|
||||
"loss": 2.159627685546875,
|
||||
"mean_token_accuracy": 0.5959913417696953,
|
||||
"num_tokens": 130613.0,
|
||||
"step": 75
|
||||
},
|
||||
{
|
||||
"entropy": 1.9658770501613616,
|
||||
"epoch": 0.17777777777777778,
|
||||
"grad_norm": 30.0,
|
||||
"learning_rate": 1.1715976331360948e-05,
|
||||
"loss": 1.9642219543457031,
|
||||
"mean_token_accuracy": 0.6274332204461097,
|
||||
"num_tokens": 174274.0,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"entropy": 1.9181486976146698,
|
||||
"epoch": 0.2222222222222222,
|
||||
"grad_norm": 23.875,
|
||||
"learning_rate": 1.4674556213017752e-05,
|
||||
"loss": 1.9006834411621094,
|
||||
"mean_token_accuracy": 0.6362838646769524,
|
||||
"num_tokens": 217619.0,
|
||||
"step": 125
|
||||
},
|
||||
{
|
||||
"entropy": 1.8227408236265183,
|
||||
"epoch": 0.26666666666666666,
|
||||
"grad_norm": 29.5,
|
||||
"learning_rate": 1.7633136094674557e-05,
|
||||
"loss": 1.834572296142578,
|
||||
"mean_token_accuracy": 0.645097343325615,
|
||||
"num_tokens": 261407.0,
|
||||
"step": 150
|
||||
},
|
||||
{
|
||||
"entropy": 1.8350859326124191,
|
||||
"epoch": 0.3111111111111111,
|
||||
"grad_norm": 25.625,
|
||||
"learning_rate": 1.999946602771351e-05,
|
||||
"loss": 1.8147303771972656,
|
||||
"mean_token_accuracy": 0.6461810152232647,
|
||||
"num_tokens": 305402.0,
|
||||
"step": 175
|
||||
},
|
||||
{
|
||||
"entropy": 1.753722729086876,
|
||||
"epoch": 0.35555555555555557,
|
||||
"grad_norm": 31.5,
|
||||
"learning_rate": 1.9980782984658682e-05,
|
||||
"loss": 1.758114013671875,
|
||||
"mean_token_accuracy": 0.653534509241581,
|
||||
"num_tokens": 348998.0,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"entropy": 1.7685393619537353,
|
||||
"epoch": 0.4,
|
||||
"grad_norm": 23.75,
|
||||
"learning_rate": 1.9935458325191365e-05,
|
||||
"loss": 1.7483135986328124,
|
||||
"mean_token_accuracy": 0.6603708426654339,
|
||||
"num_tokens": 392837.0,
|
||||
"step": 225
|
||||
},
|
||||
{
|
||||
"entropy": 1.7378248673677446,
|
||||
"epoch": 0.4444444444444444,
|
||||
"grad_norm": 26.5,
|
||||
"learning_rate": 1.9863613034027224e-05,
|
||||
"loss": 1.7749085998535157,
|
||||
"mean_token_accuracy": 0.65810049533844,
|
||||
"num_tokens": 436281.0,
|
||||
"step": 250
|
||||
},
|
||||
{
|
||||
"entropy": 1.6766968676447869,
|
||||
"epoch": 0.4888888888888889,
|
||||
"grad_norm": 23.25,
|
||||
"learning_rate": 1.9765438887171327e-05,
|
||||
"loss": 1.668856658935547,
|
||||
"mean_token_accuracy": 0.6644712510704994,
|
||||
"num_tokens": 480451.0,
|
||||
"step": 275
|
||||
},
|
||||
{
|
||||
"entropy": 1.669287955760956,
|
||||
"epoch": 0.5333333333333333,
|
||||
"grad_norm": 20.625,
|
||||
"learning_rate": 1.9641197940012136e-05,
|
||||
"loss": 1.648075714111328,
|
||||
"mean_token_accuracy": 0.6689342276751995,
|
||||
"num_tokens": 524418.0,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"entropy": 1.587462457716465,
|
||||
"epoch": 0.5777777777777777,
|
||||
"grad_norm": 21.625,
|
||||
"learning_rate": 1.9491221827819348e-05,
|
||||
"loss": 1.5674880981445312,
|
||||
"mean_token_accuracy": 0.6790469540655613,
|
||||
"num_tokens": 568093.0,
|
||||
"step": 325
|
||||
},
|
||||
{
|
||||
"entropy": 1.5877643951773643,
|
||||
"epoch": 0.6222222222222222,
|
||||
"grad_norm": 23.5,
|
||||
"learning_rate": 1.9315910880512792e-05,
|
||||
"loss": 1.5833602905273438,
|
||||
"mean_token_accuracy": 0.67303916528821,
|
||||
"num_tokens": 611875.0,
|
||||
"step": 350
|
||||
},
|
||||
{
|
||||
"entropy": 1.5628123182058333,
|
||||
"epoch": 0.6666666666666666,
|
||||
"grad_norm": 19.875,
|
||||
"learning_rate": 1.911573305406528e-05,
|
||||
"loss": 1.5389964294433593,
|
||||
"mean_token_accuracy": 0.6916262510418892,
|
||||
"num_tokens": 655258.0,
|
||||
"step": 375
|
||||
},
|
||||
{
|
||||
"entropy": 1.5605447962880135,
|
||||
"epoch": 0.7111111111111111,
|
||||
"grad_norm": 25.25,
|
||||
"learning_rate": 1.8891222681391853e-05,
|
||||
"loss": 1.5674354553222656,
|
||||
"mean_token_accuracy": 0.6811535279452801,
|
||||
"num_tokens": 698915.0,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"entropy": 1.5302811539173127,
|
||||
"epoch": 0.7555555555555555,
|
||||
"grad_norm": 21.0,
|
||||
"learning_rate": 1.8642979046059595e-05,
|
||||
"loss": 1.5149029541015624,
|
||||
"mean_token_accuracy": 0.6928434443473815,
|
||||
"num_tokens": 742608.0,
|
||||
"step": 425
|
||||
},
|
||||
{
|
||||
"entropy": 1.4984435787796975,
|
||||
"epoch": 0.8,
|
||||
"grad_norm": 20.5,
|
||||
"learning_rate": 1.8371664782625287e-05,
|
||||
"loss": 1.4783544921875,
|
||||
"mean_token_accuracy": 0.6922086542844772,
|
||||
"num_tokens": 785924.0,
|
||||
"step": 450
|
||||
},
|
||||
{
|
||||
"entropy": 1.4762829530239105,
|
||||
"epoch": 0.8444444444444444,
|
||||
"grad_norm": 22.875,
|
||||
"learning_rate": 1.8078004107870797e-05,
|
||||
"loss": 1.4516114807128906,
|
||||
"mean_token_accuracy": 0.702991289794445,
|
||||
"num_tokens": 829069.0,
|
||||
"step": 475
|
||||
},
|
||||
{
|
||||
"entropy": 1.5141191801428795,
|
||||
"epoch": 0.8888888888888888,
|
||||
"grad_norm": 20.75,
|
||||
"learning_rate": 1.7762780887657576e-05,
|
||||
"loss": 1.5113209533691405,
|
||||
"mean_token_accuracy": 0.6913984973728656,
|
||||
"num_tokens": 872824.0,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"entropy": 1.4147853454947472,
|
||||
"epoch": 0.9333333333333333,
|
||||
"grad_norm": 25.75,
|
||||
"learning_rate": 1.74268365445604e-05,
|
||||
"loss": 1.394475555419922,
|
||||
"mean_token_accuracy": 0.712423055768013,
|
||||
"num_tokens": 916212.0,
|
||||
"step": 525
|
||||
},
|
||||
{
|
||||
"entropy": 1.4362201416492462,
|
||||
"epoch": 0.9777777777777777,
|
||||
"grad_norm": 21.25,
|
||||
"learning_rate": 1.7071067811865477e-05,
|
||||
"loss": 1.4004106140136718,
|
||||
"mean_token_accuracy": 0.7106147593259812,
|
||||
"num_tokens": 959695.0,
|
||||
"step": 550
|
||||
},
|
||||
{
|
||||
"epoch": 1.0,
|
||||
"eval_entropy": 1.3886345291137696,
|
||||
"eval_loss": 1.3774118423461914,
|
||||
"eval_mean_token_accuracy": 0.7105021185874939,
|
||||
"eval_num_tokens": 981504.0,
|
||||
"eval_runtime": 28.7124,
|
||||
"eval_samples_per_second": 34.828,
|
||||
"eval_steps_per_second": 4.354,
|
||||
"step": 563
|
||||
},
|
||||
{
|
||||
"entropy": 1.3702899970266285,
|
||||
"epoch": 1.0213333333333334,
|
||||
"grad_norm": 24.375,
|
||||
"learning_rate": 1.6696424339928153e-05,
|
||||
"loss": 1.2998457336425782,
|
||||
"mean_token_accuracy": 0.7251553581077226,
|
||||
"num_tokens": 1002442.0,
|
||||
"step": 575
|
||||
},
|
||||
{
|
||||
"entropy": 1.2198174016177654,
|
||||
"epoch": 1.0657777777777777,
|
||||
"grad_norm": 19.0,
|
||||
"learning_rate": 1.6303906161279554e-05,
|
||||
"loss": 1.1578231048583985,
|
||||
"mean_token_accuracy": 0.7492367821931839,
|
||||
"num_tokens": 1045974.0,
|
||||
"step": 600
|
||||
},
|
||||
{
|
||||
"entropy": 1.2214804862439632,
|
||||
"epoch": 1.1102222222222222,
|
||||
"grad_norm": 21.5,
|
||||
"learning_rate": 1.5894561021248535e-05,
|
||||
"loss": 1.1222823333740235,
|
||||
"mean_token_accuracy": 0.7596075300872326,
|
||||
"num_tokens": 1089691.0,
|
||||
"step": 625
|
||||
},
|
||||
{
|
||||
"entropy": 1.230571178793907,
|
||||
"epoch": 1.1546666666666667,
|
||||
"grad_norm": 18.25,
|
||||
"learning_rate": 1.5469481581224274e-05,
|
||||
"loss": 1.1252061462402343,
|
||||
"mean_token_accuracy": 0.7554993903636933,
|
||||
"num_tokens": 1133544.0,
|
||||
"step": 650
|
||||
},
|
||||
{
|
||||
"entropy": 1.2035815131664276,
|
||||
"epoch": 1.199111111111111,
|
||||
"grad_norm": 17.75,
|
||||
"learning_rate": 1.5029802502024788e-05,
|
||||
"loss": 1.1525257110595704,
|
||||
"mean_token_accuracy": 0.7539819921553135,
|
||||
"num_tokens": 1176944.0,
|
||||
"step": 675
|
||||
},
|
||||
{
|
||||
"entropy": 1.1947351975739002,
|
||||
"epoch": 1.2435555555555555,
|
||||
"grad_norm": 19.5,
|
||||
"learning_rate": 1.4576697415156818e-05,
|
||||
"loss": 1.1312750244140626,
|
||||
"mean_token_accuracy": 0.755437722504139,
|
||||
"num_tokens": 1220536.0,
|
||||
"step": 700
|
||||
},
|
||||
{
|
||||
"entropy": 1.2491805362701416,
|
||||
"epoch": 1.288,
|
||||
"grad_norm": 18.625,
|
||||
"learning_rate": 1.4111375790051511e-05,
|
||||
"loss": 1.1570246124267578,
|
||||
"mean_token_accuracy": 0.7577420398592949,
|
||||
"num_tokens": 1264076.0,
|
||||
"step": 725
|
||||
},
|
||||
{
|
||||
"entropy": 1.2654732164740563,
|
||||
"epoch": 1.3324444444444445,
|
||||
"grad_norm": 20.875,
|
||||
"learning_rate": 1.3635079705638298e-05,
|
||||
"loss": 1.1855599212646484,
|
||||
"mean_token_accuracy": 0.751485439389944,
|
||||
"num_tokens": 1308034.0,
|
||||
"step": 750
|
||||
},
|
||||
{
|
||||
"entropy": 1.2152655026316643,
|
||||
"epoch": 1.3768888888888888,
|
||||
"grad_norm": 20.25,
|
||||
"learning_rate": 1.3149080534874519e-05,
|
||||
"loss": 1.1257845306396483,
|
||||
"mean_token_accuracy": 0.7632686151564121,
|
||||
"num_tokens": 1351646.0,
|
||||
"step": 775
|
||||
},
|
||||
{
|
||||
"entropy": 1.193697052896023,
|
||||
"epoch": 1.4213333333333333,
|
||||
"grad_norm": 24.75,
|
||||
"learning_rate": 1.2654675551080724e-05,
|
||||
"loss": 1.121256103515625,
|
||||
"mean_token_accuracy": 0.7563131806254387,
|
||||
"num_tokens": 1395047.0,
|
||||
"step": 800
|
||||
},
|
||||
{
|
||||
"entropy": 1.1570866696536541,
|
||||
"epoch": 1.4657777777777778,
|
||||
"grad_norm": 22.25,
|
||||
"learning_rate": 1.2153184465140413e-05,
|
||||
"loss": 1.0696445465087892,
|
||||
"mean_token_accuracy": 0.765836420059204,
|
||||
"num_tokens": 1438280.0,
|
||||
"step": 825
|
||||
},
|
||||
{
|
||||
"entropy": 1.151665413379669,
|
||||
"epoch": 1.5102222222222221,
|
||||
"grad_norm": 17.75,
|
||||
"learning_rate": 1.164594590280734e-05,
|
||||
"loss": 1.0663511657714844,
|
||||
"mean_token_accuracy": 0.7644271370768547,
|
||||
"num_tokens": 1481978.0,
|
||||
"step": 850
|
||||
},
|
||||
{
|
||||
"entropy": 1.2120365902781487,
|
||||
"epoch": 1.5546666666666666,
|
||||
"grad_norm": 17.5,
|
||||
"learning_rate": 1.1134313831523547e-05,
|
||||
"loss": 1.1270946502685546,
|
||||
"mean_token_accuracy": 0.7585732811689376,
|
||||
"num_tokens": 1525597.0,
|
||||
"step": 875
|
||||
},
|
||||
{
|
||||
"entropy": 1.151113938242197,
|
||||
"epoch": 1.5991111111111111,
|
||||
"grad_norm": 21.5,
|
||||
"learning_rate": 1.0619653946285948e-05,
|
||||
"loss": 1.0662925720214844,
|
||||
"mean_token_accuracy": 0.7686286368966102,
|
||||
"num_tokens": 1568743.0,
|
||||
"step": 900
|
||||
},
|
||||
{
|
||||
"entropy": 1.162338248193264,
|
||||
"epoch": 1.6435555555555554,
|
||||
"grad_norm": 20.625,
|
||||
"learning_rate": 1.0103340024208674e-05,
|
||||
"loss": 1.0983226013183593,
|
||||
"mean_token_accuracy": 0.7634404581785202,
|
||||
"num_tokens": 1612238.0,
|
||||
"step": 925
|
||||
},
|
||||
{
|
||||
"entropy": 1.1956017450988292,
|
||||
"epoch": 1.688,
|
||||
"grad_norm": 18.625,
|
||||
"learning_rate": 9.586750257511868e-06,
|
||||
"loss": 1.1054475402832031,
|
||||
"mean_token_accuracy": 0.7624230819940567,
|
||||
"num_tokens": 1656115.0,
|
||||
"step": 950
|
||||
},
|
||||
{
|
||||
"entropy": 1.1524758899211884,
|
||||
"epoch": 1.7324444444444445,
|
||||
"grad_norm": 20.875,
|
||||
"learning_rate": 9.0712635747253e-06,
|
||||
"loss": 1.063654327392578,
|
||||
"mean_token_accuracy": 0.7672438740730285,
|
||||
"num_tokens": 1700143.0,
|
||||
"step": 975
|
||||
},
|
||||
{
|
||||
"entropy": 1.1726488883793353,
|
||||
"epoch": 1.7768888888888887,
|
||||
"grad_norm": 18.25,
|
||||
"learning_rate": 8.558255959926533e-06,
|
||||
"loss": 1.0800904846191406,
|
||||
"mean_token_accuracy": 0.767799711227417,
|
||||
"num_tokens": 1743486.0,
|
||||
"step": 1000
|
||||
},
|
||||
{
|
||||
"entropy": 1.1595972776412964,
|
||||
"epoch": 1.8213333333333335,
|
||||
"grad_norm": 22.125,
|
||||
"learning_rate": 8.04909677983872e-06,
|
||||
"loss": 1.100532989501953,
|
||||
"mean_token_accuracy": 0.7661877936124801,
|
||||
"num_tokens": 1787577.0,
|
||||
"step": 1025
|
||||
},
|
||||
{
|
||||
"entropy": 1.1869910764694214,
|
||||
"epoch": 1.8657777777777778,
|
||||
"grad_norm": 18.625,
|
||||
"learning_rate": 7.545145128592009e-06,
|
||||
"loss": 1.1068744659423828,
|
||||
"mean_token_accuracy": 0.7658174222707749,
|
||||
"num_tokens": 1831146.0,
|
||||
"step": 1050
|
||||
},
|
||||
{
|
||||
"entropy": 1.1678849336504937,
|
||||
"epoch": 1.9102222222222223,
|
||||
"grad_norm": 20.25,
|
||||
"learning_rate": 7.0477461999055365e-06,
|
||||
"loss": 1.104344940185547,
|
||||
"mean_token_accuracy": 0.7647965340316296,
|
||||
"num_tokens": 1875315.0,
|
||||
"step": 1075
|
||||
},
|
||||
{
|
||||
"entropy": 1.2021603246033192,
|
||||
"epoch": 1.9546666666666668,
|
||||
"grad_norm": 20.75,
|
||||
"learning_rate": 6.558227696373617e-06,
|
||||
"loss": 1.1086157989501952,
|
||||
"mean_token_accuracy": 0.7716675865650177,
|
||||
"num_tokens": 1918868.0,
|
||||
"step": 1100
|
||||
},
|
||||
{
|
||||
"entropy": 1.1465536914765835,
|
||||
"epoch": 1.999111111111111,
|
||||
"grad_norm": 24.0,
|
||||
"learning_rate": 6.077896285440874e-06,
|
||||
"loss": 1.0573983001708984,
|
||||
"mean_token_accuracy": 0.7682515250146389,
|
||||
"num_tokens": 1962187.0,
|
||||
"step": 1125
|
||||
},
|
||||
{
|
||||
"epoch": 2.0,
|
||||
"eval_entropy": 1.1745851793289184,
|
||||
"eval_loss": 1.2738397121429443,
|
||||
"eval_mean_token_accuracy": 0.730101492881775,
|
||||
"eval_num_tokens": 1963008.0,
|
||||
"eval_runtime": 28.609,
|
||||
"eval_samples_per_second": 34.954,
|
||||
"eval_steps_per_second": 4.369,
|
||||
"step": 1126
|
||||
},
|
||||
{
|
||||
"entropy": 1.101325544167538,
|
||||
"epoch": 2.042666666666667,
|
||||
"grad_norm": 15.375,
|
||||
"learning_rate": 5.608034111526298e-06,
|
||||
"loss": 0.9211092376708985,
|
||||
"mean_token_accuracy": 0.8091437579417715,
|
||||
"num_tokens": 2004400.0,
|
||||
"step": 1150
|
||||
},
|
||||
{
|
||||
"entropy": 1.0406066159904004,
|
||||
"epoch": 2.087111111111111,
|
||||
"grad_norm": 20.5,
|
||||
"learning_rate": 5.149895373606405e-06,
|
||||
"loss": 0.9091825103759765,
|
||||
"mean_token_accuracy": 0.8042202299833298,
|
||||
"num_tokens": 2048143.0,
|
||||
"step": 1175
|
||||
},
|
||||
{
|
||||
"entropy": 1.0808034761250018,
|
||||
"epoch": 2.1315555555555554,
|
||||
"grad_norm": 18.0,
|
||||
"learning_rate": 4.704702977392914e-06,
|
||||
"loss": 0.9504242706298828,
|
||||
"mean_token_accuracy": 0.7970917990803719,
|
||||
"num_tokens": 2091837.0,
|
||||
"step": 1200
|
||||
},
|
||||
{
|
||||
"entropy": 1.0659918001294135,
|
||||
"epoch": 2.176,
|
||||
"grad_norm": 20.75,
|
||||
"learning_rate": 4.2736452710412645e-06,
|
||||
"loss": 0.9259893798828125,
|
||||
"mean_token_accuracy": 0.8023737239837646,
|
||||
"num_tokens": 2135628.0,
|
||||
"step": 1225
|
||||
},
|
||||
{
|
||||
"entropy": 1.0154806707799435,
|
||||
"epoch": 2.2204444444444444,
|
||||
"grad_norm": 20.0,
|
||||
"learning_rate": 3.857872873103322e-06,
|
||||
"loss": 0.8762107086181641,
|
||||
"mean_token_accuracy": 0.8123826086521149,
|
||||
"num_tokens": 2178782.0,
|
||||
"step": 1250
|
||||
},
|
||||
{
|
||||
"entropy": 1.0083947947621346,
|
||||
"epoch": 2.2648888888888887,
|
||||
"grad_norm": 21.375,
|
||||
"learning_rate": 3.4584956011913693e-06,
|
||||
"loss": 0.8710794830322266,
|
||||
"mean_token_accuracy": 0.8071741372346878,
|
||||
"num_tokens": 2222303.0,
|
||||
"step": 1275
|
||||
},
|
||||
{
|
||||
"entropy": 1.0441110938787461,
|
||||
"epoch": 2.3093333333333335,
|
||||
"grad_norm": 18.75,
|
||||
"learning_rate": 3.0765795095517026e-06,
|
||||
"loss": 0.9363404846191407,
|
||||
"mean_token_accuracy": 0.7974440288543702,
|
||||
"num_tokens": 2266083.0,
|
||||
"step": 1300
|
||||
},
|
||||
{
|
||||
"entropy": 1.0385051570832728,
|
||||
"epoch": 2.3537777777777777,
|
||||
"grad_norm": 20.625,
|
||||
"learning_rate": 2.713144043455388e-06,
|
||||
"loss": 0.9435790252685546,
|
||||
"mean_token_accuracy": 0.8013337072730065,
|
||||
"num_tokens": 2309679.0,
|
||||
"step": 1325
|
||||
},
|
||||
{
|
||||
"entropy": 1.041314078271389,
|
||||
"epoch": 2.398222222222222,
|
||||
"grad_norm": 18.25,
|
||||
"learning_rate": 2.369159318001937e-06,
|
||||
"loss": 0.9011815643310547,
|
||||
"mean_token_accuracy": 0.8010741752386094,
|
||||
"num_tokens": 2353394.0,
|
||||
"step": 1350
|
||||
},
|
||||
{
|
||||
"entropy": 1.045836333334446,
|
||||
"epoch": 2.4426666666666668,
|
||||
"grad_norm": 20.625,
|
||||
"learning_rate": 2.045543528599607e-06,
|
||||
"loss": 0.9371556091308594,
|
||||
"mean_token_accuracy": 0.8022435599565506,
|
||||
"num_tokens": 2397064.0,
|
||||
"step": 1375
|
||||
},
|
||||
{
|
||||
"entropy": 1.02559718221426,
|
||||
"epoch": 2.487111111111111,
|
||||
"grad_norm": 16.875,
|
||||
"learning_rate": 1.743160500034443e-06,
|
||||
"loss": 0.9111165618896484,
|
||||
"mean_token_accuracy": 0.8056430542469024,
|
||||
"num_tokens": 2440899.0,
|
||||
"step": 1400
|
||||
},
|
||||
{
|
||||
"entropy": 0.9850550366938114,
|
||||
"epoch": 2.5315555555555553,
|
||||
"grad_norm": 18.25,
|
||||
"learning_rate": 1.4628173806703594e-06,
|
||||
"loss": 0.8460595703125,
|
||||
"mean_token_accuracy": 0.814309296309948,
|
||||
"num_tokens": 2484527.0,
|
||||
"step": 1425
|
||||
},
|
||||
{
|
||||
"entropy": 1.0671714194118977,
|
||||
"epoch": 2.576,
|
||||
"grad_norm": 16.5,
|
||||
"learning_rate": 1.2052624879351105e-06,
|
||||
"loss": 0.9366078186035156,
|
||||
"mean_token_accuracy": 0.7982332086563111,
|
||||
"num_tokens": 2528184.0,
|
||||
"step": 1450
|
||||
},
|
||||
{
|
||||
"entropy": 1.0057188278436662,
|
||||
"epoch": 2.6204444444444444,
|
||||
"grad_norm": 18.25,
|
||||
"learning_rate": 9.711833108431234e-07,
|
||||
"loss": 0.8691239929199219,
|
||||
"mean_token_accuracy": 0.8100746890902519,
|
||||
"num_tokens": 2571871.0,
|
||||
"step": 1475
|
||||
},
|
||||
{
|
||||
"entropy": 1.0419890847057105,
|
||||
"epoch": 2.664888888888889,
|
||||
"grad_norm": 18.25,
|
||||
"learning_rate": 7.612046748871327e-07,
|
||||
"loss": 0.8980624389648437,
|
||||
"mean_token_accuracy": 0.8050528278946877,
|
||||
"num_tokens": 2615338.0,
|
||||
"step": 1500
|
||||
},
|
||||
{
|
||||
"entropy": 1.0672108069062234,
|
||||
"epoch": 2.7093333333333334,
|
||||
"grad_norm": 21.25,
|
||||
"learning_rate": 5.758870741969635e-07,
|
||||
"loss": 0.9245491790771484,
|
||||
"mean_token_accuracy": 0.8008973941206932,
|
||||
"num_tokens": 2659309.0,
|
||||
"step": 1525
|
||||
},
|
||||
{
|
||||
"entropy": 1.0332248322665691,
|
||||
"epoch": 2.7537777777777777,
|
||||
"grad_norm": 17.25,
|
||||
"learning_rate": 4.1572517541747294e-07,
|
||||
"loss": 0.9085790252685547,
|
||||
"mean_token_accuracy": 0.8022564603388309,
|
||||
"num_tokens": 2703096.0,
|
||||
"step": 1550
|
||||
},
|
||||
{
|
||||
"entropy": 1.0293236848711969,
|
||||
"epoch": 2.7982222222222224,
|
||||
"grad_norm": 18.125,
|
||||
"learning_rate": 2.811464972992195e-07,
|
||||
"loss": 0.8811993408203125,
|
||||
"mean_token_accuracy": 0.8039135798811913,
|
||||
"num_tokens": 2746647.0,
|
||||
"step": 1575
|
||||
},
|
||||
{
|
||||
"entropy": 1.0229834078252316,
|
||||
"epoch": 2.8426666666666667,
|
||||
"grad_norm": 25.625,
|
||||
"learning_rate": 1.7251026952640583e-07,
|
||||
"loss": 0.90928466796875,
|
||||
"mean_token_accuracy": 0.806032055914402,
|
||||
"num_tokens": 2790363.0,
|
||||
"step": 1600
|
||||
},
|
||||
{
|
||||
"entropy": 1.051690325587988,
|
||||
"epoch": 2.887111111111111,
|
||||
"grad_norm": 21.75,
|
||||
"learning_rate": 9.010647382825421e-08,
|
||||
"loss": 0.9022364807128906,
|
||||
"mean_token_accuracy": 0.8011369660496712,
|
||||
"num_tokens": 2834043.0,
|
||||
"step": 1625
|
||||
},
|
||||
{
|
||||
"entropy": 0.9921600778400897,
|
||||
"epoch": 2.9315555555555557,
|
||||
"grad_norm": 21.75,
|
||||
"learning_rate": 3.4155069933301535e-08,
|
||||
"loss": 0.8524517822265625,
|
||||
"mean_token_accuracy": 0.8153755596280098,
|
||||
"num_tokens": 2877282.0,
|
||||
"step": 1650
|
||||
},
|
||||
{
|
||||
"entropy": 1.0421569736301899,
|
||||
"epoch": 2.976,
|
||||
"grad_norm": 20.625,
|
||||
"learning_rate": 4.80540843283972e-09,
|
||||
"loss": 0.93547607421875,
|
||||
"mean_token_accuracy": 0.8020336532592773,
|
||||
"num_tokens": 2920814.0,
|
||||
"step": 1675
|
||||
},
|
||||
{
|
||||
"epoch": 3.0,
|
||||
"eval_entropy": 1.1009210176467896,
|
||||
"eval_loss": 1.3000445365905762,
|
||||
"eval_mean_token_accuracy": 0.7269604291915893,
|
||||
"eval_num_tokens": 2944512.0,
|
||||
"eval_runtime": 28.9685,
|
||||
"eval_samples_per_second": 34.52,
|
||||
"eval_steps_per_second": 4.315,
|
||||
"step": 1689
|
||||
}
|
||||
],
|
||||
"logging_steps": 25,
|
||||
"max_steps": 1689,
|
||||
"num_input_tokens_seen": 0,
|
||||
"num_train_epochs": 3,
|
||||
"save_steps": 500,
|
||||
"stateful_callbacks": {
|
||||
"TrainerControl": {
|
||||
"args": {
|
||||
"should_epoch_stop": false,
|
||||
"should_evaluate": false,
|
||||
"should_log": false,
|
||||
"should_save": true,
|
||||
"should_training_stop": true
|
||||
},
|
||||
"attributes": {}
|
||||
}
|
||||
},
|
||||
"total_flos": 7572413980065792.0,
|
||||
"train_batch_size": 4,
|
||||
"trial_name": null,
|
||||
"trial_params": null
|
||||
}
|
||||
3
checkpoint-1689/training_args.bin
Normal file
3
checkpoint-1689/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:355351641492ac43c5b6480ea76c78a8467059a8b861f06d7c50a5d5d9ef8de5
|
||||
size 5777
|
||||
54
checkpoint-563/chat_template.jinja
Normal file
54
checkpoint-563/chat_template.jinja
Normal file
@@ -0,0 +1,54 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- messages[0]['content'] }}
|
||||
{%- else %}
|
||||
{{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
|
||||
{%- endif %}
|
||||
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role }}
|
||||
{%- if message.content %}
|
||||
{{- '\n' + message.content }}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
57
checkpoint-563/config.json
Normal file
57
checkpoint-563/config.json
Normal file
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen2ForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": null,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 151645,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 896,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 4864,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 21,
|
||||
"model_type": "qwen2",
|
||||
"num_attention_heads": 14,
|
||||
"num_hidden_layers": 24,
|
||||
"num_key_value_heads": 2,
|
||||
"pad_token_id": 151643,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.12.1",
|
||||
"use_cache": false,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
13
checkpoint-563/generation_config.json
Normal file
13
checkpoint-563/generation_config.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"repetition_penalty": 1.1,
|
||||
"temperature": 0.7,
|
||||
"top_k": 20,
|
||||
"top_p": 0.8,
|
||||
"transformers_version": "5.12.1"
|
||||
}
|
||||
3
checkpoint-563/model.safetensors
Normal file
3
checkpoint-563/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a18c7fd953cec33a6852b165ebf0cdfe37c1cf633a5362079901cec98f2d665f
|
||||
size 988097824
|
||||
3
checkpoint-563/optimizer.pt
Normal file
3
checkpoint-563/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:0f0fbf20cf279eefdce58201a63dcf4837c626ba9b4a6ccf8b20d7744e8fa7d9
|
||||
size 1976378699
|
||||
3
checkpoint-563/rng_state.pth
Normal file
3
checkpoint-563/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:8407170b12f057bd7b1da40ade6c7756852998f6b90e57c87efcd41f170f76ea
|
||||
size 14645
|
||||
3
checkpoint-563/scheduler.pt
Normal file
3
checkpoint-563/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c768d7fb333b254d4383d7e64de54af0dba9db035f2a8ae778ecb44c349410c8
|
||||
size 1465
|
||||
3
checkpoint-563/tokenizer.json
Normal file
3
checkpoint-563/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
|
||||
size 11421892
|
||||
30
checkpoint-563/tokenizer_config.json
Normal file
30
checkpoint-563/tokenizer_config.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"backend": "tokenizers",
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"is_local": false,
|
||||
"local_files_only": false,
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
265
checkpoint-563/trainer_state.json
Normal file
265
checkpoint-563/trainer_state.json
Normal file
@@ -0,0 +1,265 @@
|
||||
{
|
||||
"best_global_step": 563,
|
||||
"best_metric": 1.3774118423461914,
|
||||
"best_model_checkpoint": "finetuned_models/FFT/checkpoint-563",
|
||||
"epoch": 1.0,
|
||||
"eval_steps": 500,
|
||||
"global_step": 563,
|
||||
"is_hyper_param_search": false,
|
||||
"is_local_process_zero": true,
|
||||
"is_world_process_zero": true,
|
||||
"log_history": [
|
||||
{
|
||||
"entropy": 1.5691047963500022,
|
||||
"epoch": 0.044444444444444446,
|
||||
"grad_norm": 61.75,
|
||||
"learning_rate": 2.840236686390533e-06,
|
||||
"loss": 2.82507080078125,
|
||||
"mean_token_accuracy": 0.5578909605741501,
|
||||
"num_tokens": 43581.0,
|
||||
"step": 25
|
||||
},
|
||||
{
|
||||
"entropy": 1.912336548268795,
|
||||
"epoch": 0.08888888888888889,
|
||||
"grad_norm": 41.25,
|
||||
"learning_rate": 5.7988165680473375e-06,
|
||||
"loss": 2.3563003540039062,
|
||||
"mean_token_accuracy": 0.5931275233626365,
|
||||
"num_tokens": 86889.0,
|
||||
"step": 50
|
||||
},
|
||||
{
|
||||
"entropy": 2.1532553315162657,
|
||||
"epoch": 0.13333333333333333,
|
||||
"grad_norm": 31.125,
|
||||
"learning_rate": 8.757396449704143e-06,
|
||||
"loss": 2.159627685546875,
|
||||
"mean_token_accuracy": 0.5959913417696953,
|
||||
"num_tokens": 130613.0,
|
||||
"step": 75
|
||||
},
|
||||
{
|
||||
"entropy": 1.9658770501613616,
|
||||
"epoch": 0.17777777777777778,
|
||||
"grad_norm": 30.0,
|
||||
"learning_rate": 1.1715976331360948e-05,
|
||||
"loss": 1.9642219543457031,
|
||||
"mean_token_accuracy": 0.6274332204461097,
|
||||
"num_tokens": 174274.0,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"entropy": 1.9181486976146698,
|
||||
"epoch": 0.2222222222222222,
|
||||
"grad_norm": 23.875,
|
||||
"learning_rate": 1.4674556213017752e-05,
|
||||
"loss": 1.9006834411621094,
|
||||
"mean_token_accuracy": 0.6362838646769524,
|
||||
"num_tokens": 217619.0,
|
||||
"step": 125
|
||||
},
|
||||
{
|
||||
"entropy": 1.8227408236265183,
|
||||
"epoch": 0.26666666666666666,
|
||||
"grad_norm": 29.5,
|
||||
"learning_rate": 1.7633136094674557e-05,
|
||||
"loss": 1.834572296142578,
|
||||
"mean_token_accuracy": 0.645097343325615,
|
||||
"num_tokens": 261407.0,
|
||||
"step": 150
|
||||
},
|
||||
{
|
||||
"entropy": 1.8350859326124191,
|
||||
"epoch": 0.3111111111111111,
|
||||
"grad_norm": 25.625,
|
||||
"learning_rate": 1.999946602771351e-05,
|
||||
"loss": 1.8147303771972656,
|
||||
"mean_token_accuracy": 0.6461810152232647,
|
||||
"num_tokens": 305402.0,
|
||||
"step": 175
|
||||
},
|
||||
{
|
||||
"entropy": 1.753722729086876,
|
||||
"epoch": 0.35555555555555557,
|
||||
"grad_norm": 31.5,
|
||||
"learning_rate": 1.9980782984658682e-05,
|
||||
"loss": 1.758114013671875,
|
||||
"mean_token_accuracy": 0.653534509241581,
|
||||
"num_tokens": 348998.0,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"entropy": 1.7685393619537353,
|
||||
"epoch": 0.4,
|
||||
"grad_norm": 23.75,
|
||||
"learning_rate": 1.9935458325191365e-05,
|
||||
"loss": 1.7483135986328124,
|
||||
"mean_token_accuracy": 0.6603708426654339,
|
||||
"num_tokens": 392837.0,
|
||||
"step": 225
|
||||
},
|
||||
{
|
||||
"entropy": 1.7378248673677446,
|
||||
"epoch": 0.4444444444444444,
|
||||
"grad_norm": 26.5,
|
||||
"learning_rate": 1.9863613034027224e-05,
|
||||
"loss": 1.7749085998535157,
|
||||
"mean_token_accuracy": 0.65810049533844,
|
||||
"num_tokens": 436281.0,
|
||||
"step": 250
|
||||
},
|
||||
{
|
||||
"entropy": 1.6766968676447869,
|
||||
"epoch": 0.4888888888888889,
|
||||
"grad_norm": 23.25,
|
||||
"learning_rate": 1.9765438887171327e-05,
|
||||
"loss": 1.668856658935547,
|
||||
"mean_token_accuracy": 0.6644712510704994,
|
||||
"num_tokens": 480451.0,
|
||||
"step": 275
|
||||
},
|
||||
{
|
||||
"entropy": 1.669287955760956,
|
||||
"epoch": 0.5333333333333333,
|
||||
"grad_norm": 20.625,
|
||||
"learning_rate": 1.9641197940012136e-05,
|
||||
"loss": 1.648075714111328,
|
||||
"mean_token_accuracy": 0.6689342276751995,
|
||||
"num_tokens": 524418.0,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"entropy": 1.587462457716465,
|
||||
"epoch": 0.5777777777777777,
|
||||
"grad_norm": 21.625,
|
||||
"learning_rate": 1.9491221827819348e-05,
|
||||
"loss": 1.5674880981445312,
|
||||
"mean_token_accuracy": 0.6790469540655613,
|
||||
"num_tokens": 568093.0,
|
||||
"step": 325
|
||||
},
|
||||
{
|
||||
"entropy": 1.5877643951773643,
|
||||
"epoch": 0.6222222222222222,
|
||||
"grad_norm": 23.5,
|
||||
"learning_rate": 1.9315910880512792e-05,
|
||||
"loss": 1.5833602905273438,
|
||||
"mean_token_accuracy": 0.67303916528821,
|
||||
"num_tokens": 611875.0,
|
||||
"step": 350
|
||||
},
|
||||
{
|
||||
"entropy": 1.5628123182058333,
|
||||
"epoch": 0.6666666666666666,
|
||||
"grad_norm": 19.875,
|
||||
"learning_rate": 1.911573305406528e-05,
|
||||
"loss": 1.5389964294433593,
|
||||
"mean_token_accuracy": 0.6916262510418892,
|
||||
"num_tokens": 655258.0,
|
||||
"step": 375
|
||||
},
|
||||
{
|
||||
"entropy": 1.5605447962880135,
|
||||
"epoch": 0.7111111111111111,
|
||||
"grad_norm": 25.25,
|
||||
"learning_rate": 1.8891222681391853e-05,
|
||||
"loss": 1.5674354553222656,
|
||||
"mean_token_accuracy": 0.6811535279452801,
|
||||
"num_tokens": 698915.0,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"entropy": 1.5302811539173127,
|
||||
"epoch": 0.7555555555555555,
|
||||
"grad_norm": 21.0,
|
||||
"learning_rate": 1.8642979046059595e-05,
|
||||
"loss": 1.5149029541015624,
|
||||
"mean_token_accuracy": 0.6928434443473815,
|
||||
"num_tokens": 742608.0,
|
||||
"step": 425
|
||||
},
|
||||
{
|
||||
"entropy": 1.4984435787796975,
|
||||
"epoch": 0.8,
|
||||
"grad_norm": 20.5,
|
||||
"learning_rate": 1.8371664782625287e-05,
|
||||
"loss": 1.4783544921875,
|
||||
"mean_token_accuracy": 0.6922086542844772,
|
||||
"num_tokens": 785924.0,
|
||||
"step": 450
|
||||
},
|
||||
{
|
||||
"entropy": 1.4762829530239105,
|
||||
"epoch": 0.8444444444444444,
|
||||
"grad_norm": 22.875,
|
||||
"learning_rate": 1.8078004107870797e-05,
|
||||
"loss": 1.4516114807128906,
|
||||
"mean_token_accuracy": 0.702991289794445,
|
||||
"num_tokens": 829069.0,
|
||||
"step": 475
|
||||
},
|
||||
{
|
||||
"entropy": 1.5141191801428795,
|
||||
"epoch": 0.8888888888888888,
|
||||
"grad_norm": 20.75,
|
||||
"learning_rate": 1.7762780887657576e-05,
|
||||
"loss": 1.5113209533691405,
|
||||
"mean_token_accuracy": 0.6913984973728656,
|
||||
"num_tokens": 872824.0,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"entropy": 1.4147853454947472,
|
||||
"epoch": 0.9333333333333333,
|
||||
"grad_norm": 25.75,
|
||||
"learning_rate": 1.74268365445604e-05,
|
||||
"loss": 1.394475555419922,
|
||||
"mean_token_accuracy": 0.712423055768013,
|
||||
"num_tokens": 916212.0,
|
||||
"step": 525
|
||||
},
|
||||
{
|
||||
"entropy": 1.4362201416492462,
|
||||
"epoch": 0.9777777777777777,
|
||||
"grad_norm": 21.25,
|
||||
"learning_rate": 1.7071067811865477e-05,
|
||||
"loss": 1.4004106140136718,
|
||||
"mean_token_accuracy": 0.7106147593259812,
|
||||
"num_tokens": 959695.0,
|
||||
"step": 550
|
||||
},
|
||||
{
|
||||
"epoch": 1.0,
|
||||
"eval_entropy": 1.3886345291137696,
|
||||
"eval_loss": 1.3774118423461914,
|
||||
"eval_mean_token_accuracy": 0.7105021185874939,
|
||||
"eval_num_tokens": 981504.0,
|
||||
"eval_runtime": 28.7124,
|
||||
"eval_samples_per_second": 34.828,
|
||||
"eval_steps_per_second": 4.354,
|
||||
"step": 563
|
||||
}
|
||||
],
|
||||
"logging_steps": 25,
|
||||
"max_steps": 1689,
|
||||
"num_input_tokens_seen": 0,
|
||||
"num_train_epochs": 3,
|
||||
"save_steps": 500,
|
||||
"stateful_callbacks": {
|
||||
"TrainerControl": {
|
||||
"args": {
|
||||
"should_epoch_stop": false,
|
||||
"should_evaluate": false,
|
||||
"should_log": false,
|
||||
"should_save": true,
|
||||
"should_training_stop": false
|
||||
},
|
||||
"attributes": {}
|
||||
}
|
||||
},
|
||||
"total_flos": 2522133763928064.0,
|
||||
"train_batch_size": 4,
|
||||
"trial_name": null,
|
||||
"trial_params": null
|
||||
}
|
||||
3
checkpoint-563/training_args.bin
Normal file
3
checkpoint-563/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:355351641492ac43c5b6480ea76c78a8467059a8b861f06d7c50a5d5d9ef8de5
|
||||
size 5777
|
||||
57
config.json
Normal file
57
config.json
Normal file
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen2ForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": null,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 151645,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 896,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 4864,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 21,
|
||||
"model_type": "qwen2",
|
||||
"num_attention_heads": 14,
|
||||
"num_hidden_layers": 24,
|
||||
"num_key_value_heads": 2,
|
||||
"pad_token_id": 151643,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.12.1",
|
||||
"use_cache": false,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
1001
fft_predictions.csv
Normal file
1001
fft_predictions.csv
Normal file
File diff suppressed because it is too large
Load Diff
13
generation_config.json
Normal file
13
generation_config.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"repetition_penalty": 1.1,
|
||||
"temperature": 0.7,
|
||||
"top_k": 20,
|
||||
"top_p": 0.8,
|
||||
"transformers_version": "5.12.1"
|
||||
}
|
||||
52
metrics.json
Normal file
52
metrics.json
Normal file
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"epoch_end_validation": [
|
||||
{
|
||||
"split": "validation_epoch_1",
|
||||
"num_examples": 1000,
|
||||
"num_errors": 0,
|
||||
"functional_match_rate": 0.2653061224489796,
|
||||
"structural_match_rate": 0.052,
|
||||
"syntax_validity_rate": 0.302,
|
||||
"base_command_match_rate": 0.35,
|
||||
"epoch": 1.0
|
||||
},
|
||||
{
|
||||
"split": "validation_epoch_2",
|
||||
"num_examples": 1000,
|
||||
"num_errors": 0,
|
||||
"functional_match_rate": 0.3142857142857143,
|
||||
"structural_match_rate": 0.067,
|
||||
"syntax_validity_rate": 0.277,
|
||||
"base_command_match_rate": 0.405,
|
||||
"epoch": 2.0
|
||||
},
|
||||
{
|
||||
"split": "validation_epoch_3",
|
||||
"num_examples": 1000,
|
||||
"num_errors": 0,
|
||||
"functional_match_rate": 0.3333333333333333,
|
||||
"structural_match_rate": 0.077,
|
||||
"syntax_validity_rate": 0.279,
|
||||
"base_command_match_rate": 0.404,
|
||||
"epoch": 3.0
|
||||
}
|
||||
],
|
||||
"validation_final": {
|
||||
"split": "validation_final",
|
||||
"num_examples": 1000,
|
||||
"num_errors": 0,
|
||||
"functional_match_rate": 0.28865979381443296,
|
||||
"structural_match_rate": 0.081,
|
||||
"syntax_validity_rate": 0.283,
|
||||
"base_command_match_rate": 0.395
|
||||
},
|
||||
"test": {
|
||||
"split": "test",
|
||||
"num_examples": 1000,
|
||||
"num_errors": 0,
|
||||
"functional_match_rate": 0.25396825396825395,
|
||||
"structural_match_rate": 0.072,
|
||||
"syntax_validity_rate": 0.401,
|
||||
"base_command_match_rate": 0.34
|
||||
}
|
||||
}
|
||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:b5f8b69d19a2f71c729d2eb592d3ed38ab299591b0ac2f2a2620f21d1a501c74
|
||||
size 988097824
|
||||
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
|
||||
size 11421892
|
||||
30
tokenizer_config.json
Normal file
30
tokenizer_config.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"backend": "tokenizers",
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"is_local": false,
|
||||
"local_files_only": false,
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:355351641492ac43c5b6480ea76c78a8467059a8b861f06d7c50a5d5d9ef8de5
|
||||
size 5777
|
||||
1001
validation_predictions.csv
Normal file
1001
validation_predictions.csv
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user