初始化项目，由ModelHub XC社区提供模型
Model: terasut/sft-qwen2.5-1.5b-instruct-eff32 Source: Original Platform
This commit is contained in:
ModelHub XC
39
.gitattributes
vendored
Normal file
39
.gitattributes
vendored
Normal file
@@ -0,0 +1,39 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
checkpoint-1750/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
checkpoint-2625/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
checkpoint-875/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
57
README.md
Normal file
57
README.md
Normal file
@@ -0,0 +1,57 @@
|
||||
---
|
||||
library_name: transformers
|
||||
model_name: sft-qwen2.5-1.5b-instruct-eff32
|
||||
tags:
|
||||
- generated_from_trainer
|
||||
- trl
|
||||
- sft
|
||||
licence: license
|
||||
---
|
||||
|
||||
# Model Card for sft-qwen2.5-1.5b-instruct-eff32
|
||||
|
||||
This model is a fine-tuned version of [None](https://huggingface.co/None).
|
||||
It has been trained using [TRL](https://github.com/huggingface/trl).
|
||||
|
||||
## Quick start
|
||||
|
||||
```python
|
||||
from transformers import pipeline
|
||||
|
||||
question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
|
||||
generator = pipeline("text-generation", model="None", device="cuda")
|
||||
output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
|
||||
print(output["generated_text"])
|
||||
```
|
||||
|
||||
## Training procedure
|
||||
|
||||
[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/terasut-workspace/ai-patternrecog/runs/uafxmgbk)
|
||||
|
||||
|
||||
|
||||
This model was trained with SFT.
|
||||
|
||||
### Framework versions
|
||||
|
||||
- TRL: 1.1.0
|
||||
- Transformers: 5.5.4
|
||||
- Pytorch: 2.10.0+cu128
|
||||
- Datasets: 4.8.4
|
||||
- Tokenizers: 0.22.2
|
||||
|
||||
## Citations
|
||||
|
||||
|
||||
|
||||
Cite TRL as:
|
||||
|
||||
```bibtex
|
||||
@software{vonwerra2020trl,
|
||||
title = {{TRL: Transformers Reinforcement Learning}},
|
||||
author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
|
||||
license = {Apache-2.0},
|
||||
url = {https://github.com/huggingface/trl},
|
||||
year = {2020}
|
||||
}
|
||||
```
|
||||
54
chat_template.jinja
Normal file
54
chat_template.jinja
Normal file
@@ -0,0 +1,54 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- messages[0]['content'] }}
|
||||
{%- else %}
|
||||
{{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
|
||||
{%- endif %}
|
||||
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role }}
|
||||
{%- if message.content %}
|
||||
{{- '\n' + message.content }}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
54
checkpoint-1750/chat_template.jinja
Normal file
54
checkpoint-1750/chat_template.jinja
Normal file
@@ -0,0 +1,54 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- messages[0]['content'] }}
|
||||
{%- else %}
|
||||
{{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
|
||||
{%- endif %}
|
||||
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role }}
|
||||
{%- if message.content %}
|
||||
{{- '\n' + message.content }}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
61
checkpoint-1750/config.json
Normal file
61
checkpoint-1750/config.json
Normal file
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen2ForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": null,
|
||||
"dtype": "float32",
|
||||
"eos_token_id": 151645,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 1536,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 8960,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 21,
|
||||
"model_type": "qwen2",
|
||||
"num_attention_heads": 12,
|
||||
"num_hidden_layers": 28,
|
||||
"num_key_value_heads": 2,
|
||||
"pad_token_id": 151643,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.5.4",
|
||||
"use_cache": false,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
14
checkpoint-1750/generation_config.json
Normal file
14
checkpoint-1750/generation_config.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"repetition_penalty": 1.1,
|
||||
"temperature": 0.7,
|
||||
"top_k": 20,
|
||||
"top_p": 0.8,
|
||||
"transformers_version": "5.5.4",
|
||||
"use_cache": false
|
||||
}
|
||||
3
checkpoint-1750/model.safetensors
Normal file
3
checkpoint-1750/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:48b69ec3864095fcce9513b3ba341b6b0554f1dbd84c9e1393f1501b41140167
|
||||
size 6174895536
|
||||
3
checkpoint-1750/optimizer.pt
Normal file
3
checkpoint-1750/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:58d85847e17209c7e695400aaa9053f7244ca0755eebcaf61f982b682dcdf98f
|
||||
size 12350013801
|
||||
3
checkpoint-1750/rng_state.pth
Normal file
3
checkpoint-1750/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878
|
||||
size 14645
|
||||
3
checkpoint-1750/scheduler.pt
Normal file
3
checkpoint-1750/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fc53ebd3c8ca395477da766ae7fa14a1cd87577954defcd87cfec2f1d6fd424a
|
||||
size 1465
|
||||
3
checkpoint-1750/tokenizer.json
Normal file
3
checkpoint-1750/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
|
||||
size 11421892
|
||||
30
checkpoint-1750/tokenizer_config.json
Normal file
30
checkpoint-1750/tokenizer_config.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"backend": "tokenizers",
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"is_local": true,
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"padding_side": "left",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
1784
checkpoint-1750/trainer_state.json
Normal file
1784
checkpoint-1750/trainer_state.json
Normal file
File diff suppressed because it is too large
Load Diff
3
checkpoint-1750/training_args.bin
Normal file
3
checkpoint-1750/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:accc301f3ad366c553289bf58cab2de1488f42639e4826bf1d58853e8aa93ad2
|
||||
size 5841
|
||||
54
checkpoint-2625/chat_template.jinja
Normal file
54
checkpoint-2625/chat_template.jinja
Normal file
@@ -0,0 +1,54 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- messages[0]['content'] }}
|
||||
{%- else %}
|
||||
{{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
|
||||
{%- endif %}
|
||||
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role }}
|
||||
{%- if message.content %}
|
||||
{{- '\n' + message.content }}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
61
checkpoint-2625/config.json
Normal file
61
checkpoint-2625/config.json
Normal file
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen2ForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": null,
|
||||
"dtype": "float32",
|
||||
"eos_token_id": 151645,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 1536,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 8960,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 21,
|
||||
"model_type": "qwen2",
|
||||
"num_attention_heads": 12,
|
||||
"num_hidden_layers": 28,
|
||||
"num_key_value_heads": 2,
|
||||
"pad_token_id": 151643,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.5.4",
|
||||
"use_cache": false,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
14
checkpoint-2625/generation_config.json
Normal file
14
checkpoint-2625/generation_config.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"repetition_penalty": 1.1,
|
||||
"temperature": 0.7,
|
||||
"top_k": 20,
|
||||
"top_p": 0.8,
|
||||
"transformers_version": "5.5.4",
|
||||
"use_cache": false
|
||||
}
|
||||
3
checkpoint-2625/model.safetensors
Normal file
3
checkpoint-2625/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ebaae81661331d6e842dce368a7337b16107ab56f651ceefde1fb347cf1acc6b
|
||||
size 6174895536
|
||||
3
checkpoint-2625/optimizer.pt
Normal file
3
checkpoint-2625/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:cfcbc6d3617a3524bbccc7923eb522af5fff4270d019e45031f1787810b9f939
|
||||
size 12350013801
|
||||
3
checkpoint-2625/rng_state.pth
Normal file
3
checkpoint-2625/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f4a9f217e852f439efa6bd32fde98d6867f11aa6ea13ddc021ba10af6a0b0934
|
||||
size 14645
|
||||
3
checkpoint-2625/scheduler.pt
Normal file
3
checkpoint-2625/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a90cf32bcaa40d2760ac49ca5a2167553d01347c3172274302ea44fcf542f183
|
||||
size 1465
|
||||
3
checkpoint-2625/tokenizer.json
Normal file
3
checkpoint-2625/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
|
||||
size 11421892
|
||||
30
checkpoint-2625/tokenizer_config.json
Normal file
30
checkpoint-2625/tokenizer_config.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"backend": "tokenizers",
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"is_local": true,
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"padding_side": "left",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
2654
checkpoint-2625/trainer_state.json
Normal file
2654
checkpoint-2625/trainer_state.json
Normal file
File diff suppressed because it is too large
Load Diff
3
checkpoint-2625/training_args.bin
Normal file
3
checkpoint-2625/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:accc301f3ad366c553289bf58cab2de1488f42639e4826bf1d58853e8aa93ad2
|
||||
size 5841
|
||||
54
checkpoint-875/chat_template.jinja
Normal file
54
checkpoint-875/chat_template.jinja
Normal file
@@ -0,0 +1,54 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- messages[0]['content'] }}
|
||||
{%- else %}
|
||||
{{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
|
||||
{%- endif %}
|
||||
{{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role }}
|
||||
{%- if message.content %}
|
||||
{{- '\n' + message.content }}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
61
checkpoint-875/config.json
Normal file
61
checkpoint-875/config.json
Normal file
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen2ForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": null,
|
||||
"dtype": "float32",
|
||||
"eos_token_id": 151645,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 1536,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 8960,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 21,
|
||||
"model_type": "qwen2",
|
||||
"num_attention_heads": 12,
|
||||
"num_hidden_layers": 28,
|
||||
"num_key_value_heads": 2,
|
||||
"pad_token_id": 151643,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.5.4",
|
||||
"use_cache": false,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
14
checkpoint-875/generation_config.json
Normal file
14
checkpoint-875/generation_config.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"repetition_penalty": 1.1,
|
||||
"temperature": 0.7,
|
||||
"top_k": 20,
|
||||
"top_p": 0.8,
|
||||
"transformers_version": "5.5.4",
|
||||
"use_cache": false
|
||||
}
|
||||
3
checkpoint-875/model.safetensors
Normal file
3
checkpoint-875/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5001fa75b8fdbcc4529f9a419f3d620fa25de4f77ccd9814b055fb9b148981dd
|
||||
size 6174895536
|
||||
3
checkpoint-875/optimizer.pt
Normal file
3
checkpoint-875/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d73b7576d5803b41450dbe747cfc0ab9a345c7016cfb28b8315e041802a28cc8
|
||||
size 12350013801
|
||||
3
checkpoint-875/rng_state.pth
Normal file
3
checkpoint-875/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:61c19bab1174704a4a4441475683bf1270277af15d2e2c95e964789128e482c4
|
||||
size 14645
|
||||
3
checkpoint-875/scheduler.pt
Normal file
3
checkpoint-875/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:977dc1268f934bf99945ed0df8810e94a86fb8f53f0a4e7c78e4ae26e606231a
|
||||
size 1465
|
||||
3
checkpoint-875/tokenizer.json
Normal file
3
checkpoint-875/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
|
||||
size 11421892
|
||||
30
checkpoint-875/tokenizer_config.json
Normal file
30
checkpoint-875/tokenizer_config.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"backend": "tokenizers",
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"is_local": true,
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"padding_side": "left",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
904
checkpoint-875/trainer_state.json
Normal file
904
checkpoint-875/trainer_state.json
Normal file
@@ -0,0 +1,904 @@
|
||||
{
|
||||
"best_global_step": null,
|
||||
"best_metric": null,
|
||||
"best_model_checkpoint": null,
|
||||
"epoch": 1.0,
|
||||
"eval_steps": 500,
|
||||
"global_step": 875,
|
||||
"is_hyper_param_search": false,
|
||||
"is_local_process_zero": true,
|
||||
"is_world_process_zero": true,
|
||||
"log_history": [
|
||||
{
|
||||
"entropy": 1.6696195736527444,
|
||||
"epoch": 0.011428571428571429,
|
||||
"grad_norm": 3.7833805084228516,
|
||||
"learning_rate": 1.9931428571428572e-05,
|
||||
"loss": 1.3203315734863281,
|
||||
"mean_token_accuracy": 0.7056225836277008,
|
||||
"num_tokens": 157266.0,
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"entropy": 1.7283091336488723,
|
||||
"epoch": 0.022857142857142857,
|
||||
"grad_norm": 3.4964022636413574,
|
||||
"learning_rate": 1.9855238095238097e-05,
|
||||
"loss": 1.4036809921264648,
|
||||
"mean_token_accuracy": 0.6812524735927582,
|
||||
"num_tokens": 300430.0,
|
||||
"step": 20
|
||||
},
|
||||
{
|
||||
"entropy": 1.7663277059793472,
|
||||
"epoch": 0.03428571428571429,
|
||||
"grad_norm": 4.590687274932861,
|
||||
"learning_rate": 1.977904761904762e-05,
|
||||
"loss": 1.319040012359619,
|
||||
"mean_token_accuracy": 0.6928375385701656,
|
||||
"num_tokens": 456129.0,
|
||||
"step": 30
|
||||
},
|
||||
{
|
||||
"entropy": 1.7699106186628342,
|
||||
"epoch": 0.045714285714285714,
|
||||
"grad_norm": 3.2531967163085938,
|
||||
"learning_rate": 1.9702857142857144e-05,
|
||||
"loss": 1.3776089668273925,
|
||||
"mean_token_accuracy": 0.686311125010252,
|
||||
"num_tokens": 615179.0,
|
||||
"step": 40
|
||||
},
|
||||
{
|
||||
"entropy": 1.7856220349669456,
|
||||
"epoch": 0.05714285714285714,
|
||||
"grad_norm": 3.356832981109619,
|
||||
"learning_rate": 1.9626666666666666e-05,
|
||||
"loss": 1.32177791595459,
|
||||
"mean_token_accuracy": 0.6952089086174965,
|
||||
"num_tokens": 769672.0,
|
||||
"step": 50
|
||||
},
|
||||
{
|
||||
"entropy": 1.7628433778882027,
|
||||
"epoch": 0.06857142857142857,
|
||||
"grad_norm": 3.965970754623413,
|
||||
"learning_rate": 1.955047619047619e-05,
|
||||
"loss": 1.338463306427002,
|
||||
"mean_token_accuracy": 0.6953431364148855,
|
||||
"num_tokens": 918402.0,
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"entropy": 1.7276440173387528,
|
||||
"epoch": 0.08,
|
||||
"grad_norm": 3.4119951725006104,
|
||||
"learning_rate": 1.9474285714285717e-05,
|
||||
"loss": 1.2838379859924316,
|
||||
"mean_token_accuracy": 0.6987539391964674,
|
||||
"num_tokens": 1076598.0,
|
||||
"step": 70
|
||||
},
|
||||
{
|
||||
"entropy": 1.748648078739643,
|
||||
"epoch": 0.09142857142857143,
|
||||
"grad_norm": 3.6497740745544434,
|
||||
"learning_rate": 1.9398095238095242e-05,
|
||||
"loss": 1.3314701080322267,
|
||||
"mean_token_accuracy": 0.695335328578949,
|
||||
"num_tokens": 1228084.0,
|
||||
"step": 80
|
||||
},
|
||||
{
|
||||
"entropy": 1.7111000671982766,
|
||||
"epoch": 0.10285714285714286,
|
||||
"grad_norm": 3.3122711181640625,
|
||||
"learning_rate": 1.9321904761904764e-05,
|
||||
"loss": 1.3027889251708984,
|
||||
"mean_token_accuracy": 0.6987856134772301,
|
||||
"num_tokens": 1386707.0,
|
||||
"step": 90
|
||||
},
|
||||
{
|
||||
"entropy": 1.7278845444321633,
|
||||
"epoch": 0.11428571428571428,
|
||||
"grad_norm": 3.183835744857788,
|
||||
"learning_rate": 1.924571428571429e-05,
|
||||
"loss": 1.282834243774414,
|
||||
"mean_token_accuracy": 0.6966647863388061,
|
||||
"num_tokens": 1535406.0,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"entropy": 1.7132800757884978,
|
||||
"epoch": 0.12571428571428572,
|
||||
"grad_norm": 3.4031126499176025,
|
||||
"learning_rate": 1.916952380952381e-05,
|
||||
"loss": 1.2409746170043945,
|
||||
"mean_token_accuracy": 0.7101543735712766,
|
||||
"num_tokens": 1680743.0,
|
||||
"step": 110
|
||||
},
|
||||
{
|
||||
"entropy": 1.7373003125190736,
|
||||
"epoch": 0.13714285714285715,
|
||||
"grad_norm": 3.4207799434661865,
|
||||
"learning_rate": 1.9093333333333336e-05,
|
||||
"loss": 1.2909335136413573,
|
||||
"mean_token_accuracy": 0.701117354631424,
|
||||
"num_tokens": 1834014.0,
|
||||
"step": 120
|
||||
},
|
||||
{
|
||||
"entropy": 1.7092810094356536,
|
||||
"epoch": 0.14857142857142858,
|
||||
"grad_norm": 3.3323278427124023,
|
||||
"learning_rate": 1.9017142857142858e-05,
|
||||
"loss": 1.3009337425231933,
|
||||
"mean_token_accuracy": 0.6907773412764072,
|
||||
"num_tokens": 1989349.0,
|
||||
"step": 130
|
||||
},
|
||||
{
|
||||
"entropy": 1.7240261450409888,
|
||||
"epoch": 0.16,
|
||||
"grad_norm": 2.9499671459198,
|
||||
"learning_rate": 1.8940952380952383e-05,
|
||||
"loss": 1.2700564384460449,
|
||||
"mean_token_accuracy": 0.7077916815876961,
|
||||
"num_tokens": 2139614.0,
|
||||
"step": 140
|
||||
},
|
||||
{
|
||||
"entropy": 1.655298152565956,
|
||||
"epoch": 0.17142857142857143,
|
||||
"grad_norm": 3.1871607303619385,
|
||||
"learning_rate": 1.8864761904761905e-05,
|
||||
"loss": 1.2251598358154296,
|
||||
"mean_token_accuracy": 0.7074991308152676,
|
||||
"num_tokens": 2293879.0,
|
||||
"step": 150
|
||||
},
|
||||
{
|
||||
"entropy": 1.665707252919674,
|
||||
"epoch": 0.18285714285714286,
|
||||
"grad_norm": 3.290358304977417,
|
||||
"learning_rate": 1.878857142857143e-05,
|
||||
"loss": 1.2641545295715333,
|
||||
"mean_token_accuracy": 0.7021077655255794,
|
||||
"num_tokens": 2446447.0,
|
||||
"step": 160
|
||||
},
|
||||
{
|
||||
"entropy": 1.7259689077734948,
|
||||
"epoch": 0.19428571428571428,
|
||||
"grad_norm": 3.1401703357696533,
|
||||
"learning_rate": 1.871238095238095e-05,
|
||||
"loss": 1.315788745880127,
|
||||
"mean_token_accuracy": 0.6926948539912701,
|
||||
"num_tokens": 2595389.0,
|
||||
"step": 170
|
||||
},
|
||||
{
|
||||
"entropy": 1.7142695009708404,
|
||||
"epoch": 0.2057142857142857,
|
||||
"grad_norm": 3.4904985427856445,
|
||||
"learning_rate": 1.8636190476190477e-05,
|
||||
"loss": 1.2755393028259276,
|
||||
"mean_token_accuracy": 0.7040164832025766,
|
||||
"num_tokens": 2745608.0,
|
||||
"step": 180
|
||||
},
|
||||
{
|
||||
"entropy": 1.6242914512753486,
|
||||
"epoch": 0.21714285714285714,
|
||||
"grad_norm": 2.949880361557007,
|
||||
"learning_rate": 1.8560000000000002e-05,
|
||||
"loss": 1.190676498413086,
|
||||
"mean_token_accuracy": 0.7185754984617233,
|
||||
"num_tokens": 2894625.0,
|
||||
"step": 190
|
||||
},
|
||||
{
|
||||
"entropy": 1.7364101350307464,
|
||||
"epoch": 0.22857142857142856,
|
||||
"grad_norm": 3.125112533569336,
|
||||
"learning_rate": 1.8483809523809527e-05,
|
||||
"loss": 1.2999603271484375,
|
||||
"mean_token_accuracy": 0.6976742841303348,
|
||||
"num_tokens": 3046487.0,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"entropy": 1.6772031649947166,
|
||||
"epoch": 0.24,
|
||||
"grad_norm": 2.813882350921631,
|
||||
"learning_rate": 1.840761904761905e-05,
|
||||
"loss": 1.2316917419433593,
|
||||
"mean_token_accuracy": 0.7120772495865821,
|
||||
"num_tokens": 3196704.0,
|
||||
"step": 210
|
||||
},
|
||||
{
|
||||
"entropy": 1.6737122178077697,
|
||||
"epoch": 0.25142857142857145,
|
||||
"grad_norm": 2.8218936920166016,
|
||||
"learning_rate": 1.8331428571428574e-05,
|
||||
"loss": 1.274891471862793,
|
||||
"mean_token_accuracy": 0.6991564333438873,
|
||||
"num_tokens": 3348365.0,
|
||||
"step": 220
|
||||
},
|
||||
{
|
||||
"entropy": 1.6344200998544693,
|
||||
"epoch": 0.26285714285714284,
|
||||
"grad_norm": 2.9688668251037598,
|
||||
"learning_rate": 1.8255238095238096e-05,
|
||||
"loss": 1.2082077026367188,
|
||||
"mean_token_accuracy": 0.7141494430601597,
|
||||
"num_tokens": 3509464.0,
|
||||
"step": 230
|
||||
},
|
||||
{
|
||||
"entropy": 1.6166698306798934,
|
||||
"epoch": 0.2742857142857143,
|
||||
"grad_norm": 3.007690668106079,
|
||||
"learning_rate": 1.817904761904762e-05,
|
||||
"loss": 1.1915247917175293,
|
||||
"mean_token_accuracy": 0.7145515654236079,
|
||||
"num_tokens": 3661779.0,
|
||||
"step": 240
|
||||
},
|
||||
{
|
||||
"entropy": 1.6418054014444352,
|
||||
"epoch": 0.2857142857142857,
|
||||
"grad_norm": 2.9565091133117676,
|
||||
"learning_rate": 1.8102857142857143e-05,
|
||||
"loss": 1.2041254043579102,
|
||||
"mean_token_accuracy": 0.7189058735966682,
|
||||
"num_tokens": 3809611.0,
|
||||
"step": 250
|
||||
},
|
||||
{
|
||||
"entropy": 1.6628487035632133,
|
||||
"epoch": 0.29714285714285715,
|
||||
"grad_norm": 2.7851827144622803,
|
||||
"learning_rate": 1.8026666666666668e-05,
|
||||
"loss": 1.2013505935668944,
|
||||
"mean_token_accuracy": 0.7156483136117459,
|
||||
"num_tokens": 3960501.0,
|
||||
"step": 260
|
||||
},
|
||||
{
|
||||
"entropy": 1.6495753720402717,
|
||||
"epoch": 0.30857142857142855,
|
||||
"grad_norm": 3.0212292671203613,
|
||||
"learning_rate": 1.7950476190476193e-05,
|
||||
"loss": 1.2240056991577148,
|
||||
"mean_token_accuracy": 0.7174453124403953,
|
||||
"num_tokens": 4107783.0,
|
||||
"step": 270
|
||||
},
|
||||
{
|
||||
"entropy": 1.693277756869793,
|
||||
"epoch": 0.32,
|
||||
"grad_norm": 2.9716391563415527,
|
||||
"learning_rate": 1.7874285714285715e-05,
|
||||
"loss": 1.207960319519043,
|
||||
"mean_token_accuracy": 0.7157351300120354,
|
||||
"num_tokens": 4252335.0,
|
||||
"step": 280
|
||||
},
|
||||
{
|
||||
"entropy": 1.6619624510407447,
|
||||
"epoch": 0.3314285714285714,
|
||||
"grad_norm": 2.851775646209717,
|
||||
"learning_rate": 1.779809523809524e-05,
|
||||
"loss": 1.248055648803711,
|
||||
"mean_token_accuracy": 0.7128231346607208,
|
||||
"num_tokens": 4407579.0,
|
||||
"step": 290
|
||||
},
|
||||
{
|
||||
"entropy": 1.6617254123091698,
|
||||
"epoch": 0.34285714285714286,
|
||||
"grad_norm": 3.254999876022339,
|
||||
"learning_rate": 1.7721904761904762e-05,
|
||||
"loss": 1.184535598754883,
|
||||
"mean_token_accuracy": 0.7231362499296665,
|
||||
"num_tokens": 4551517.0,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"entropy": 1.5988039791584014,
|
||||
"epoch": 0.35428571428571426,
|
||||
"grad_norm": 3.110564708709717,
|
||||
"learning_rate": 1.7645714285714287e-05,
|
||||
"loss": 1.148481559753418,
|
||||
"mean_token_accuracy": 0.7262889519333839,
|
||||
"num_tokens": 4708257.0,
|
||||
"step": 310
|
||||
},
|
||||
{
|
||||
"entropy": 1.6530713498592378,
|
||||
"epoch": 0.3657142857142857,
|
||||
"grad_norm": 3.1247048377990723,
|
||||
"learning_rate": 1.7569523809523812e-05,
|
||||
"loss": 1.2834845542907716,
|
||||
"mean_token_accuracy": 0.702876628190279,
|
||||
"num_tokens": 4859075.0,
|
||||
"step": 320
|
||||
},
|
||||
{
|
||||
"entropy": 1.6123583316802979,
|
||||
"epoch": 0.37714285714285717,
|
||||
"grad_norm": 2.90073823928833,
|
||||
"learning_rate": 1.7493333333333334e-05,
|
||||
"loss": 1.1132370948791503,
|
||||
"mean_token_accuracy": 0.7313019946217537,
|
||||
"num_tokens": 5011395.0,
|
||||
"step": 330
|
||||
},
|
||||
{
|
||||
"entropy": 1.618579277396202,
|
||||
"epoch": 0.38857142857142857,
|
||||
"grad_norm": 2.936490058898926,
|
||||
"learning_rate": 1.741714285714286e-05,
|
||||
"loss": 1.2263901710510254,
|
||||
"mean_token_accuracy": 0.7069775387644768,
|
||||
"num_tokens": 5174272.0,
|
||||
"step": 340
|
||||
},
|
||||
{
|
||||
"entropy": 1.663301782310009,
|
||||
"epoch": 0.4,
|
||||
"grad_norm": 3.241185426712036,
|
||||
"learning_rate": 1.734095238095238e-05,
|
||||
"loss": 1.2628487586975097,
|
||||
"mean_token_accuracy": 0.7058808147907257,
|
||||
"num_tokens": 5327670.0,
|
||||
"step": 350
|
||||
},
|
||||
{
|
||||
"entropy": 1.6857714764773846,
|
||||
"epoch": 0.4114285714285714,
|
||||
"grad_norm": 2.9059698581695557,
|
||||
"learning_rate": 1.7264761904761906e-05,
|
||||
"loss": 1.2412065505981444,
|
||||
"mean_token_accuracy": 0.7062458634376526,
|
||||
"num_tokens": 5476064.0,
|
||||
"step": 360
|
||||
},
|
||||
{
|
||||
"entropy": 1.6253925159573555,
|
||||
"epoch": 0.4228571428571429,
|
||||
"grad_norm": 2.9899251461029053,
|
||||
"learning_rate": 1.718857142857143e-05,
|
||||
"loss": 1.210451316833496,
|
||||
"mean_token_accuracy": 0.7089586913585663,
|
||||
"num_tokens": 5636828.0,
|
||||
"step": 370
|
||||
},
|
||||
{
|
||||
"entropy": 1.5880890995264054,
|
||||
"epoch": 0.4342857142857143,
|
||||
"grad_norm": 3.0461251735687256,
|
||||
"learning_rate": 1.7112380952380953e-05,
|
||||
"loss": 1.1992740631103516,
|
||||
"mean_token_accuracy": 0.7144776731729507,
|
||||
"num_tokens": 5788462.0,
|
||||
"step": 380
|
||||
},
|
||||
{
|
||||
"entropy": 1.6384857930243015,
|
||||
"epoch": 0.44571428571428573,
|
||||
"grad_norm": 2.8842580318450928,
|
||||
"learning_rate": 1.703619047619048e-05,
|
||||
"loss": 1.2124631881713868,
|
||||
"mean_token_accuracy": 0.7115042388439179,
|
||||
"num_tokens": 5943108.0,
|
||||
"step": 390
|
||||
},
|
||||
{
|
||||
"entropy": 1.5625910967588426,
|
||||
"epoch": 0.45714285714285713,
|
||||
"grad_norm": 3.0432047843933105,
|
||||
"learning_rate": 1.696e-05,
|
||||
"loss": 1.1515995025634767,
|
||||
"mean_token_accuracy": 0.716732618585229,
|
||||
"num_tokens": 6098679.0,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"entropy": 1.6026007533073425,
|
||||
"epoch": 0.4685714285714286,
|
||||
"grad_norm": 2.781804323196411,
|
||||
"learning_rate": 1.6883809523809525e-05,
|
||||
"loss": 1.180698299407959,
|
||||
"mean_token_accuracy": 0.7132757268846035,
|
||||
"num_tokens": 6250301.0,
|
||||
"step": 410
|
||||
},
|
||||
{
|
||||
"entropy": 1.5140968441963196,
|
||||
"epoch": 0.48,
|
||||
"grad_norm": 2.9592671394348145,
|
||||
"learning_rate": 1.6807619047619047e-05,
|
||||
"loss": 1.1643908500671387,
|
||||
"mean_token_accuracy": 0.7223096333444119,
|
||||
"num_tokens": 6407356.0,
|
||||
"step": 420
|
||||
},
|
||||
{
|
||||
"entropy": 1.6122839108109475,
|
||||
"epoch": 0.49142857142857144,
|
||||
"grad_norm": 3.2809886932373047,
|
||||
"learning_rate": 1.6731428571428572e-05,
|
||||
"loss": 1.2397551536560059,
|
||||
"mean_token_accuracy": 0.707809828966856,
|
||||
"num_tokens": 6559819.0,
|
||||
"step": 430
|
||||
},
|
||||
{
|
||||
"entropy": 1.603949698805809,
|
||||
"epoch": 0.5028571428571429,
|
||||
"grad_norm": 2.721527338027954,
|
||||
"learning_rate": 1.6655238095238098e-05,
|
||||
"loss": 1.1895122528076172,
|
||||
"mean_token_accuracy": 0.7227885220199823,
|
||||
"num_tokens": 6716227.0,
|
||||
"step": 440
|
||||
},
|
||||
{
|
||||
"entropy": 1.582472313940525,
|
||||
"epoch": 0.5142857142857142,
|
||||
"grad_norm": 2.8015570640563965,
|
||||
"learning_rate": 1.6579047619047623e-05,
|
||||
"loss": 1.1144920349121095,
|
||||
"mean_token_accuracy": 0.7284132238477469,
|
||||
"num_tokens": 6867486.0,
|
||||
"step": 450
|
||||
},
|
||||
{
|
||||
"entropy": 1.6025653272867202,
|
||||
"epoch": 0.5257142857142857,
|
||||
"grad_norm": 3.1553401947021484,
|
||||
"learning_rate": 1.6502857142857145e-05,
|
||||
"loss": 1.1947656631469727,
|
||||
"mean_token_accuracy": 0.7160489454865455,
|
||||
"num_tokens": 7012179.0,
|
||||
"step": 460
|
||||
},
|
||||
{
|
||||
"entropy": 1.5280416823923588,
|
||||
"epoch": 0.5371428571428571,
|
||||
"grad_norm": 2.97623348236084,
|
||||
"learning_rate": 1.642666666666667e-05,
|
||||
"loss": 1.149429702758789,
|
||||
"mean_token_accuracy": 0.7256167627871036,
|
||||
"num_tokens": 7171638.0,
|
||||
"step": 470
|
||||
},
|
||||
{
|
||||
"entropy": 1.6620354026556015,
|
||||
"epoch": 0.5485714285714286,
|
||||
"grad_norm": 3.1873958110809326,
|
||||
"learning_rate": 1.635047619047619e-05,
|
||||
"loss": 1.2588795661926269,
|
||||
"mean_token_accuracy": 0.707894566655159,
|
||||
"num_tokens": 7322167.0,
|
||||
"step": 480
|
||||
},
|
||||
{
|
||||
"entropy": 1.581991147994995,
|
||||
"epoch": 0.56,
|
||||
"grad_norm": 3.385500907897949,
|
||||
"learning_rate": 1.6274285714285717e-05,
|
||||
"loss": 1.2005508422851563,
|
||||
"mean_token_accuracy": 0.7142980419099331,
|
||||
"num_tokens": 7466661.0,
|
||||
"step": 490
|
||||
},
|
||||
{
|
||||
"entropy": 1.6193878114223481,
|
||||
"epoch": 0.5714285714285714,
|
||||
"grad_norm": 2.9256980419158936,
|
||||
"learning_rate": 1.619809523809524e-05,
|
||||
"loss": 1.1964147567749024,
|
||||
"mean_token_accuracy": 0.715008432790637,
|
||||
"num_tokens": 7620522.0,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"entropy": 1.560711294412613,
|
||||
"epoch": 0.5828571428571429,
|
||||
"grad_norm": 2.548938035964966,
|
||||
"learning_rate": 1.6121904761904764e-05,
|
||||
"loss": 1.174423599243164,
|
||||
"mean_token_accuracy": 0.7205559216439724,
|
||||
"num_tokens": 7772467.0,
|
||||
"step": 510
|
||||
},
|
||||
{
|
||||
"entropy": 1.6210713252425193,
|
||||
"epoch": 0.5942857142857143,
|
||||
"grad_norm": 2.6339938640594482,
|
||||
"learning_rate": 1.6045714285714286e-05,
|
||||
"loss": 1.153661060333252,
|
||||
"mean_token_accuracy": 0.7224138833582401,
|
||||
"num_tokens": 7922030.0,
|
||||
"step": 520
|
||||
},
|
||||
{
|
||||
"entropy": 1.5822248332202435,
|
||||
"epoch": 0.6057142857142858,
|
||||
"grad_norm": 2.4934332370758057,
|
||||
"learning_rate": 1.596952380952381e-05,
|
||||
"loss": 1.1586255073547362,
|
||||
"mean_token_accuracy": 0.7230928719043732,
|
||||
"num_tokens": 8067955.0,
|
||||
"step": 530
|
||||
},
|
||||
{
|
||||
"entropy": 1.5642932564020158,
|
||||
"epoch": 0.6171428571428571,
|
||||
"grad_norm": 2.985651969909668,
|
||||
"learning_rate": 1.5893333333333333e-05,
|
||||
"loss": 1.1338199615478515,
|
||||
"mean_token_accuracy": 0.7240421302616596,
|
||||
"num_tokens": 8226369.0,
|
||||
"step": 540
|
||||
},
|
||||
{
|
||||
"entropy": 1.6006409794092178,
|
||||
"epoch": 0.6285714285714286,
|
||||
"grad_norm": 2.6047215461730957,
|
||||
"learning_rate": 1.5817142857142858e-05,
|
||||
"loss": 1.155271625518799,
|
||||
"mean_token_accuracy": 0.7230217695236206,
|
||||
"num_tokens": 8376285.0,
|
||||
"step": 550
|
||||
},
|
||||
{
|
||||
"entropy": 1.5791691079735757,
|
||||
"epoch": 0.64,
|
||||
"grad_norm": 2.855867385864258,
|
||||
"learning_rate": 1.5740952380952383e-05,
|
||||
"loss": 1.1597973823547363,
|
||||
"mean_token_accuracy": 0.7209223508834839,
|
||||
"num_tokens": 8530322.0,
|
||||
"step": 560
|
||||
},
|
||||
{
|
||||
"entropy": 1.6016968488693237,
|
||||
"epoch": 0.6514285714285715,
|
||||
"grad_norm": 3.205124855041504,
|
||||
"learning_rate": 1.5664761904761908e-05,
|
||||
"loss": 1.1405302047729493,
|
||||
"mean_token_accuracy": 0.7231174889951945,
|
||||
"num_tokens": 8676991.0,
|
||||
"step": 570
|
||||
},
|
||||
{
|
||||
"entropy": 1.5366713739931583,
|
||||
"epoch": 0.6628571428571428,
|
||||
"grad_norm": 3.1047000885009766,
|
||||
"learning_rate": 1.558857142857143e-05,
|
||||
"loss": 1.1293843269348145,
|
||||
"mean_token_accuracy": 0.7271328650414943,
|
||||
"num_tokens": 8833861.0,
|
||||
"step": 580
|
||||
},
|
||||
{
|
||||
"entropy": 1.5866449058055878,
|
||||
"epoch": 0.6742857142857143,
|
||||
"grad_norm": 2.884098529815674,
|
||||
"learning_rate": 1.5512380952380955e-05,
|
||||
"loss": 1.2017478942871094,
|
||||
"mean_token_accuracy": 0.716648469120264,
|
||||
"num_tokens": 8983438.0,
|
||||
"step": 590
|
||||
},
|
||||
{
|
||||
"entropy": 1.610584917664528,
|
||||
"epoch": 0.6857142857142857,
|
||||
"grad_norm": 2.7602009773254395,
|
||||
"learning_rate": 1.5436190476190477e-05,
|
||||
"loss": 1.1553681373596192,
|
||||
"mean_token_accuracy": 0.72271120660007,
|
||||
"num_tokens": 9138958.0,
|
||||
"step": 600
|
||||
},
|
||||
{
|
||||
"entropy": 1.5292311601340771,
|
||||
"epoch": 0.6971428571428572,
|
||||
"grad_norm": 2.8975393772125244,
|
||||
"learning_rate": 1.5360000000000002e-05,
|
||||
"loss": 1.1547229766845704,
|
||||
"mean_token_accuracy": 0.7226926662027836,
|
||||
"num_tokens": 9290314.0,
|
||||
"step": 610
|
||||
},
|
||||
{
|
||||
"entropy": 1.5577459454536438,
|
||||
"epoch": 0.7085714285714285,
|
||||
"grad_norm": 2.5054473876953125,
|
||||
"learning_rate": 1.5283809523809524e-05,
|
||||
"loss": 1.1239413261413573,
|
||||
"mean_token_accuracy": 0.7288053043186664,
|
||||
"num_tokens": 9442584.0,
|
||||
"step": 620
|
||||
},
|
||||
{
|
||||
"entropy": 1.6545007571578025,
|
||||
"epoch": 0.72,
|
||||
"grad_norm": 3.156757354736328,
|
||||
"learning_rate": 1.5207619047619049e-05,
|
||||
"loss": 1.2162582397460937,
|
||||
"mean_token_accuracy": 0.7100601557642221,
|
||||
"num_tokens": 9589146.0,
|
||||
"step": 630
|
||||
},
|
||||
{
|
||||
"entropy": 1.6293067336082458,
|
||||
"epoch": 0.7314285714285714,
|
||||
"grad_norm": 2.7666168212890625,
|
||||
"learning_rate": 1.5131428571428572e-05,
|
||||
"loss": 1.1888948440551759,
|
||||
"mean_token_accuracy": 0.7159062504768372,
|
||||
"num_tokens": 9736056.0,
|
||||
"step": 640
|
||||
},
|
||||
{
|
||||
"entropy": 1.5786321811378001,
|
||||
"epoch": 0.7428571428571429,
|
||||
"grad_norm": 2.6019463539123535,
|
||||
"learning_rate": 1.5055238095238096e-05,
|
||||
"loss": 1.0923637390136718,
|
||||
"mean_token_accuracy": 0.7382036492228508,
|
||||
"num_tokens": 9898410.0,
|
||||
"step": 650
|
||||
},
|
||||
{
|
||||
"entropy": 1.574275678396225,
|
||||
"epoch": 0.7542857142857143,
|
||||
"grad_norm": 2.7830586433410645,
|
||||
"learning_rate": 1.497904761904762e-05,
|
||||
"loss": 1.1398842811584473,
|
||||
"mean_token_accuracy": 0.7246641829609871,
|
||||
"num_tokens": 10051154.0,
|
||||
"step": 660
|
||||
},
|
||||
{
|
||||
"entropy": 1.5554455459117889,
|
||||
"epoch": 0.7657142857142857,
|
||||
"grad_norm": 2.8458149433135986,
|
||||
"learning_rate": 1.4902857142857143e-05,
|
||||
"loss": 1.139054298400879,
|
||||
"mean_token_accuracy": 0.726585789769888,
|
||||
"num_tokens": 10199110.0,
|
||||
"step": 670
|
||||
},
|
||||
{
|
||||
"entropy": 1.6094272822141646,
|
||||
"epoch": 0.7771428571428571,
|
||||
"grad_norm": 2.9835047721862793,
|
||||
"learning_rate": 1.4826666666666666e-05,
|
||||
"loss": 1.1978397369384766,
|
||||
"mean_token_accuracy": 0.7123216778039932,
|
||||
"num_tokens": 10344927.0,
|
||||
"step": 680
|
||||
},
|
||||
{
|
||||
"entropy": 1.5142188727855683,
|
||||
"epoch": 0.7885714285714286,
|
||||
"grad_norm": 2.7492172718048096,
|
||||
"learning_rate": 1.475047619047619e-05,
|
||||
"loss": 1.048453140258789,
|
||||
"mean_token_accuracy": 0.7415693461894989,
|
||||
"num_tokens": 10505422.0,
|
||||
"step": 690
|
||||
},
|
||||
{
|
||||
"entropy": 1.633454330265522,
|
||||
"epoch": 0.8,
|
||||
"grad_norm": 2.645536184310913,
|
||||
"learning_rate": 1.4674285714285717e-05,
|
||||
"loss": 1.1423879623413087,
|
||||
"mean_token_accuracy": 0.7272885538637638,
|
||||
"num_tokens": 10652367.0,
|
||||
"step": 700
|
||||
},
|
||||
{
|
||||
"entropy": 1.4862130410969256,
|
||||
"epoch": 0.8114285714285714,
|
||||
"grad_norm": 2.473473310470581,
|
||||
"learning_rate": 1.459809523809524e-05,
|
||||
"loss": 1.019435214996338,
|
||||
"mean_token_accuracy": 0.7454098269343377,
|
||||
"num_tokens": 10812282.0,
|
||||
"step": 710
|
||||
},
|
||||
{
|
||||
"entropy": 1.541966749727726,
|
||||
"epoch": 0.8228571428571428,
|
||||
"grad_norm": 2.5247135162353516,
|
||||
"learning_rate": 1.4521904761904764e-05,
|
||||
"loss": 1.0726137161254883,
|
||||
"mean_token_accuracy": 0.7371681958436966,
|
||||
"num_tokens": 10967866.0,
|
||||
"step": 720
|
||||
},
|
||||
{
|
||||
"entropy": 1.5219792507588863,
|
||||
"epoch": 0.8342857142857143,
|
||||
"grad_norm": 2.35876202583313,
|
||||
"learning_rate": 1.4445714285714287e-05,
|
||||
"loss": 1.0680880546569824,
|
||||
"mean_token_accuracy": 0.7386457361280918,
|
||||
"num_tokens": 11120412.0,
|
||||
"step": 730
|
||||
},
|
||||
{
|
||||
"entropy": 1.506186830997467,
|
||||
"epoch": 0.8457142857142858,
|
||||
"grad_norm": 2.9589080810546875,
|
||||
"learning_rate": 1.436952380952381e-05,
|
||||
"loss": 1.0974149703979492,
|
||||
"mean_token_accuracy": 0.7378807857632637,
|
||||
"num_tokens": 11277244.0,
|
||||
"step": 740
|
||||
},
|
||||
{
|
||||
"entropy": 1.569691103696823,
|
||||
"epoch": 0.8571428571428571,
|
||||
"grad_norm": 2.5306074619293213,
|
||||
"learning_rate": 1.4293333333333334e-05,
|
||||
"loss": 1.177268123626709,
|
||||
"mean_token_accuracy": 0.7250622183084487,
|
||||
"num_tokens": 11436007.0,
|
||||
"step": 750
|
||||
},
|
||||
{
|
||||
"entropy": 1.5297324053943158,
|
||||
"epoch": 0.8685714285714285,
|
||||
"grad_norm": 2.7663497924804688,
|
||||
"learning_rate": 1.4217142857142858e-05,
|
||||
"loss": 1.1369609832763672,
|
||||
"mean_token_accuracy": 0.7221393447369338,
|
||||
"num_tokens": 11587615.0,
|
||||
"step": 760
|
||||
},
|
||||
{
|
||||
"entropy": 1.4932486459612846,
|
||||
"epoch": 0.88,
|
||||
"grad_norm": 2.5511255264282227,
|
||||
"learning_rate": 1.4140952380952381e-05,
|
||||
"loss": 1.1008304595947265,
|
||||
"mean_token_accuracy": 0.7303555272519588,
|
||||
"num_tokens": 11747686.0,
|
||||
"step": 770
|
||||
},
|
||||
{
|
||||
"entropy": 1.5960537254810334,
|
||||
"epoch": 0.8914285714285715,
|
||||
"grad_norm": 2.450721025466919,
|
||||
"learning_rate": 1.4064761904761905e-05,
|
||||
"loss": 1.1708695411682128,
|
||||
"mean_token_accuracy": 0.7219238199293614,
|
||||
"num_tokens": 11904676.0,
|
||||
"step": 780
|
||||
},
|
||||
{
|
||||
"entropy": 1.5482905827462674,
|
||||
"epoch": 0.9028571428571428,
|
||||
"grad_norm": 3.1114189624786377,
|
||||
"learning_rate": 1.398857142857143e-05,
|
||||
"loss": 1.184025478363037,
|
||||
"mean_token_accuracy": 0.720519745349884,
|
||||
"num_tokens": 12059825.0,
|
||||
"step": 790
|
||||
},
|
||||
{
|
||||
"entropy": 1.5766600042581558,
|
||||
"epoch": 0.9142857142857143,
|
||||
"grad_norm": 3.0541417598724365,
|
||||
"learning_rate": 1.3912380952380953e-05,
|
||||
"loss": 1.2167075157165528,
|
||||
"mean_token_accuracy": 0.7153365164995193,
|
||||
"num_tokens": 12218813.0,
|
||||
"step": 800
|
||||
},
|
||||
{
|
||||
"entropy": 1.5715911239385605,
|
||||
"epoch": 0.9257142857142857,
|
||||
"grad_norm": 2.4761741161346436,
|
||||
"learning_rate": 1.3836190476190477e-05,
|
||||
"loss": 1.0981733322143554,
|
||||
"mean_token_accuracy": 0.7329004533588886,
|
||||
"num_tokens": 12366807.0,
|
||||
"step": 810
|
||||
},
|
||||
{
|
||||
"entropy": 1.5434480041265488,
|
||||
"epoch": 0.9371428571428572,
|
||||
"grad_norm": 3.157036304473877,
|
||||
"learning_rate": 1.376e-05,
|
||||
"loss": 1.0953783988952637,
|
||||
"mean_token_accuracy": 0.7374757573008537,
|
||||
"num_tokens": 12519277.0,
|
||||
"step": 820
|
||||
},
|
||||
{
|
||||
"entropy": 1.5731646195054054,
|
||||
"epoch": 0.9485714285714286,
|
||||
"grad_norm": 2.9288859367370605,
|
||||
"learning_rate": 1.3683809523809526e-05,
|
||||
"loss": 1.1861271858215332,
|
||||
"mean_token_accuracy": 0.7202281527221203,
|
||||
"num_tokens": 12668916.0,
|
||||
"step": 830
|
||||
},
|
||||
{
|
||||
"entropy": 1.5766675099730492,
|
||||
"epoch": 0.96,
|
||||
"grad_norm": 2.3330445289611816,
|
||||
"learning_rate": 1.3607619047619049e-05,
|
||||
"loss": 1.1634528160095214,
|
||||
"mean_token_accuracy": 0.7218175616115332,
|
||||
"num_tokens": 12824349.0,
|
||||
"step": 840
|
||||
},
|
||||
{
|
||||
"entropy": 1.5784073442220687,
|
||||
"epoch": 0.9714285714285714,
|
||||
"grad_norm": 2.8480734825134277,
|
||||
"learning_rate": 1.3531428571428573e-05,
|
||||
"loss": 1.1402252197265625,
|
||||
"mean_token_accuracy": 0.7281969003379345,
|
||||
"num_tokens": 12974198.0,
|
||||
"step": 850
|
||||
},
|
||||
{
|
||||
"entropy": 1.5565635196864605,
|
||||
"epoch": 0.9828571428571429,
|
||||
"grad_norm": 3.2169361114501953,
|
||||
"learning_rate": 1.3455238095238096e-05,
|
||||
"loss": 1.0919082641601563,
|
||||
"mean_token_accuracy": 0.7359561800956727,
|
||||
"num_tokens": 13125877.0,
|
||||
"step": 860
|
||||
},
|
||||
{
|
||||
"entropy": 1.6499854236841203,
|
||||
"epoch": 0.9942857142857143,
|
||||
"grad_norm": 2.59342622756958,
|
||||
"learning_rate": 1.3379047619047621e-05,
|
||||
"loss": 1.1824271202087402,
|
||||
"mean_token_accuracy": 0.7206701695919037,
|
||||
"num_tokens": 13270434.0,
|
||||
"step": 870
|
||||
}
|
||||
],
|
||||
"logging_steps": 10,
|
||||
"max_steps": 2625,
|
||||
"num_input_tokens_seen": 0,
|
||||
"num_train_epochs": 3,
|
||||
"save_steps": 500,
|
||||
"stateful_callbacks": {
|
||||
"TrainerControl": {
|
||||
"args": {
|
||||
"should_epoch_stop": false,
|
||||
"should_evaluate": false,
|
||||
"should_log": false,
|
||||
"should_save": true,
|
||||
"should_training_stop": false
|
||||
},
|
||||
"attributes": {}
|
||||
}
|
||||
},
|
||||
"total_flos": 1.6345415171049062e+17,
|
||||
"train_batch_size": 4,
|
||||
"trial_name": null,
|
||||
"trial_params": null
|
||||
}
|
||||
3
checkpoint-875/training_args.bin
Normal file
3
checkpoint-875/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:accc301f3ad366c553289bf58cab2de1488f42639e4826bf1d58853e8aa93ad2
|
||||
size 5841
|
||||
61
config.json
Normal file
61
config.json
Normal file
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen2ForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": null,
|
||||
"dtype": "float32",
|
||||
"eos_token_id": 151645,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 1536,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 8960,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 21,
|
||||
"model_type": "qwen2",
|
||||
"num_attention_heads": 12,
|
||||
"num_hidden_layers": 28,
|
||||
"num_key_value_heads": 2,
|
||||
"pad_token_id": 151643,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.5.4",
|
||||
"use_cache": false,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
14
generation_config.json
Normal file
14
generation_config.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"repetition_penalty": 1.1,
|
||||
"temperature": 0.7,
|
||||
"top_k": 20,
|
||||
"top_p": 0.8,
|
||||
"transformers_version": "5.5.4",
|
||||
"use_cache": false
|
||||
}
|
||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ebaae81661331d6e842dce368a7337b16107ab56f651ceefde1fb347cf1acc6b
|
||||
size 6174895536
|
||||
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
|
||||
size 11421892
|
||||
30
tokenizer_config.json
Normal file
30
tokenizer_config.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"backend": "tokenizers",
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"is_local": true,
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"padding_side": "left",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3a93e1c3fa00eeef707a0efd45019cc65de48c174d2ed52e764742abf7f3aae8
|
||||
size 5841
|
||||
Reference in New Issue
Block a user