初始化项目，由ModelHub XC社区提供模型
Model: Janushi/FoodExtract-gemma-3-270m-fine-tune-v1 Source: Original Platform
This commit is contained in:
ModelHub XC
40
.gitattributes
vendored
Normal file
40
.gitattributes
vendored
Normal file
@@ -0,0 +1,40 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
checkpoint-142/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
checkpoint-213/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
checkpoint-284/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
checkpoint-71/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
58
README.md
Normal file
58
README.md
Normal file
@@ -0,0 +1,58 @@
|
||||
---
|
||||
base_model: google/gemma-3-270m-it
|
||||
library_name: transformers
|
||||
model_name: checkpoint_models
|
||||
tags:
|
||||
- generated_from_trainer
|
||||
- sft
|
||||
- trl
|
||||
licence: license
|
||||
---
|
||||
|
||||
# Model Card for checkpoint_models
|
||||
|
||||
This model is a fine-tuned version of [google/gemma-3-270m-it](https://huggingface.co/google/gemma-3-270m-it).
|
||||
It has been trained using [TRL](https://github.com/huggingface/trl).
|
||||
|
||||
## Quick start
|
||||
|
||||
```python
|
||||
from transformers import pipeline
|
||||
|
||||
question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
|
||||
generator = pipeline("text-generation", model="None", device="cuda")
|
||||
output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
|
||||
print(output["generated_text"])
|
||||
```
|
||||
|
||||
## Training procedure
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
This model was trained with SFT.
|
||||
|
||||
### Framework versions
|
||||
|
||||
- TRL: 0.29.1
|
||||
- Transformers: 5.0.0
|
||||
- Pytorch: 2.10.0+cu128
|
||||
- Datasets: 4.0.0
|
||||
- Tokenizers: 0.22.2
|
||||
|
||||
## Citations
|
||||
|
||||
|
||||
|
||||
Cite TRL as:
|
||||
|
||||
```bibtex
|
||||
@software{vonwerra2020trl,
|
||||
title = {{TRL: Transformers Reinforcement Learning}},
|
||||
author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
|
||||
license = {Apache-2.0},
|
||||
url = {https://github.com/huggingface/trl},
|
||||
year = {2020}
|
||||
}
|
||||
```
|
||||
47
chat_template.jinja
Normal file
47
chat_template.jinja
Normal file
@@ -0,0 +1,47 @@
|
||||
{{ bos_token }}
|
||||
{%- if messages[0]['role'] == 'system' -%}
|
||||
{%- if messages[0]['content'] is string -%}
|
||||
{%- set first_user_prefix = messages[0]['content'] + '
|
||||
|
||||
' -%}
|
||||
{%- else -%}
|
||||
{%- set first_user_prefix = messages[0]['content'][0]['text'] + '
|
||||
|
||||
' -%}
|
||||
{%- endif -%}
|
||||
{%- set loop_messages = messages[1:] -%}
|
||||
{%- else -%}
|
||||
{%- set first_user_prefix = "" -%}
|
||||
{%- set loop_messages = messages -%}
|
||||
{%- endif -%}
|
||||
{%- for message in loop_messages -%}
|
||||
{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
|
||||
{{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
|
||||
{%- endif -%}
|
||||
{%- if (message['role'] == 'assistant') -%}
|
||||
{%- set role = "model" -%}
|
||||
{%- else -%}
|
||||
{%- set role = message['role'] -%}
|
||||
{%- endif -%}
|
||||
{{ '<start_of_turn>' + role + '
|
||||
' + (first_user_prefix if loop.first else "") }}
|
||||
{%- if message['content'] is string -%}
|
||||
{{ message['content'] | trim }}
|
||||
{%- elif message['content'] is iterable -%}
|
||||
{%- for item in message['content'] -%}
|
||||
{%- if item['type'] == 'image' -%}
|
||||
{{ '<start_of_image>' }}
|
||||
{%- elif item['type'] == 'text' -%}
|
||||
{{ item['text'] | trim }}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- else -%}
|
||||
{{ raise_exception("Invalid content type") }}
|
||||
{%- endif -%}
|
||||
{{ '<end_of_turn>
|
||||
' }}
|
||||
{%- endfor -%}
|
||||
{%- if add_generation_prompt -%}
|
||||
{{'<start_of_turn>model
|
||||
'}}
|
||||
{%- endif -%}
|
||||
47
checkpoint-142/chat_template.jinja
Normal file
47
checkpoint-142/chat_template.jinja
Normal file
@@ -0,0 +1,47 @@
|
||||
{{ bos_token }}
|
||||
{%- if messages[0]['role'] == 'system' -%}
|
||||
{%- if messages[0]['content'] is string -%}
|
||||
{%- set first_user_prefix = messages[0]['content'] + '
|
||||
|
||||
' -%}
|
||||
{%- else -%}
|
||||
{%- set first_user_prefix = messages[0]['content'][0]['text'] + '
|
||||
|
||||
' -%}
|
||||
{%- endif -%}
|
||||
{%- set loop_messages = messages[1:] -%}
|
||||
{%- else -%}
|
||||
{%- set first_user_prefix = "" -%}
|
||||
{%- set loop_messages = messages -%}
|
||||
{%- endif -%}
|
||||
{%- for message in loop_messages -%}
|
||||
{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
|
||||
{{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
|
||||
{%- endif -%}
|
||||
{%- if (message['role'] == 'assistant') -%}
|
||||
{%- set role = "model" -%}
|
||||
{%- else -%}
|
||||
{%- set role = message['role'] -%}
|
||||
{%- endif -%}
|
||||
{{ '<start_of_turn>' + role + '
|
||||
' + (first_user_prefix if loop.first else "") }}
|
||||
{%- if message['content'] is string -%}
|
||||
{{ message['content'] | trim }}
|
||||
{%- elif message['content'] is iterable -%}
|
||||
{%- for item in message['content'] -%}
|
||||
{%- if item['type'] == 'image' -%}
|
||||
{{ '<start_of_image>' }}
|
||||
{%- elif item['type'] == 'text' -%}
|
||||
{{ item['text'] | trim }}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- else -%}
|
||||
{{ raise_exception("Invalid content type") }}
|
||||
{%- endif -%}
|
||||
{{ '<end_of_turn>
|
||||
' }}
|
||||
{%- endfor -%}
|
||||
{%- if add_generation_prompt -%}
|
||||
{{'<start_of_turn>model
|
||||
'}}
|
||||
{%- endif -%}
|
||||
62
checkpoint-142/config.json
Normal file
62
checkpoint-142/config.json
Normal file
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"_sliding_window_pattern": 6,
|
||||
"architectures": [
|
||||
"Gemma3ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attn_logit_softcapping": null,
|
||||
"bos_token_id": 2,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 1,
|
||||
"final_logit_softcapping": null,
|
||||
"head_dim": 256,
|
||||
"hidden_activation": "gelu_pytorch_tanh",
|
||||
"hidden_size": 640,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 2048,
|
||||
"layer_types": [
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"model_type": "gemma3_text",
|
||||
"num_attention_heads": 4,
|
||||
"num_hidden_layers": 18,
|
||||
"num_key_value_heads": 1,
|
||||
"pad_token_id": 0,
|
||||
"query_pre_attn_scalar": 256,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"full_attention": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_attention": {
|
||||
"rope_theta": 10000.0,
|
||||
"rope_type": "default"
|
||||
}
|
||||
},
|
||||
"sliding_window": 512,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.0.0",
|
||||
"use_bidirectional_attention": false,
|
||||
"use_cache": false,
|
||||
"vocab_size": 262144
|
||||
}
|
||||
13
checkpoint-142/generation_config.json
Normal file
13
checkpoint-142/generation_config.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"bos_token_id": 2,
|
||||
"cache_implementation": "hybrid",
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
1,
|
||||
106
|
||||
],
|
||||
"pad_token_id": 0,
|
||||
"top_k": 64,
|
||||
"top_p": 0.95,
|
||||
"transformers_version": "5.0.0"
|
||||
}
|
||||
3
checkpoint-142/model.safetensors
Normal file
3
checkpoint-142/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f3c02138bcd0c3740b3bc32e9796591fca892c30cd2670d290ac0a17b6106bbb
|
||||
size 536223056
|
||||
3
checkpoint-142/optimizer.pt
Normal file
3
checkpoint-142/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:4170dff0a8aba276a89e01b9c401196d3d7a191402b469cc8fc360fc7f5eadb1
|
||||
size 1072591115
|
||||
3
checkpoint-142/rng_state.pth
Normal file
3
checkpoint-142/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a8e2011629d8bed3ef560fa11175cac55684c4e12a72634bb24abf767b6c7399
|
||||
size 14645
|
||||
3
checkpoint-142/scheduler.pt
Normal file
3
checkpoint-142/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:2732da9c9500b70ace14309058979c65a5e31cf20be217688967b97f7bd768e2
|
||||
size 1465
|
||||
3
checkpoint-142/tokenizer.json
Normal file
3
checkpoint-142/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497
|
||||
size 33384443
|
||||
24
checkpoint-142/tokenizer_config.json
Normal file
24
checkpoint-142/tokenizer_config.json
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"backend": "tokenizers",
|
||||
"boi_token": "<start_of_image>",
|
||||
"bos_token": "<bos>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eoi_token": "<end_of_image>",
|
||||
"eos_token": "<eos>",
|
||||
"image_token": "<image_soft_token>",
|
||||
"is_local": false,
|
||||
"mask_token": "<mask>",
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"model_specific_special_tokens": {
|
||||
"boi_token": "<start_of_image>",
|
||||
"eoi_token": "<end_of_image>",
|
||||
"image_token": "<image_soft_token>"
|
||||
},
|
||||
"pad_token": "<pad>",
|
||||
"padding_side": "left",
|
||||
"sp_model_kwargs": null,
|
||||
"spaces_between_special_tokens": false,
|
||||
"tokenizer_class": "GemmaTokenizer",
|
||||
"unk_token": "<unk>",
|
||||
"use_default_system_prompt": false
|
||||
}
|
||||
1476
checkpoint-142/trainer_state.json
Normal file
1476
checkpoint-142/trainer_state.json
Normal file
File diff suppressed because it is too large
Load Diff
3
checkpoint-142/training_args.bin
Normal file
3
checkpoint-142/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f426a6e6806cf7029aec0a6fb169b02e14452d298ed1e630560f4ded93439cb9
|
||||
size 5585
|
||||
47
checkpoint-213/chat_template.jinja
Normal file
47
checkpoint-213/chat_template.jinja
Normal file
@@ -0,0 +1,47 @@
|
||||
{{ bos_token }}
|
||||
{%- if messages[0]['role'] == 'system' -%}
|
||||
{%- if messages[0]['content'] is string -%}
|
||||
{%- set first_user_prefix = messages[0]['content'] + '
|
||||
|
||||
' -%}
|
||||
{%- else -%}
|
||||
{%- set first_user_prefix = messages[0]['content'][0]['text'] + '
|
||||
|
||||
' -%}
|
||||
{%- endif -%}
|
||||
{%- set loop_messages = messages[1:] -%}
|
||||
{%- else -%}
|
||||
{%- set first_user_prefix = "" -%}
|
||||
{%- set loop_messages = messages -%}
|
||||
{%- endif -%}
|
||||
{%- for message in loop_messages -%}
|
||||
{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
|
||||
{{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
|
||||
{%- endif -%}
|
||||
{%- if (message['role'] == 'assistant') -%}
|
||||
{%- set role = "model" -%}
|
||||
{%- else -%}
|
||||
{%- set role = message['role'] -%}
|
||||
{%- endif -%}
|
||||
{{ '<start_of_turn>' + role + '
|
||||
' + (first_user_prefix if loop.first else "") }}
|
||||
{%- if message['content'] is string -%}
|
||||
{{ message['content'] | trim }}
|
||||
{%- elif message['content'] is iterable -%}
|
||||
{%- for item in message['content'] -%}
|
||||
{%- if item['type'] == 'image' -%}
|
||||
{{ '<start_of_image>' }}
|
||||
{%- elif item['type'] == 'text' -%}
|
||||
{{ item['text'] | trim }}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- else -%}
|
||||
{{ raise_exception("Invalid content type") }}
|
||||
{%- endif -%}
|
||||
{{ '<end_of_turn>
|
||||
' }}
|
||||
{%- endfor -%}
|
||||
{%- if add_generation_prompt -%}
|
||||
{{'<start_of_turn>model
|
||||
'}}
|
||||
{%- endif -%}
|
||||
62
checkpoint-213/config.json
Normal file
62
checkpoint-213/config.json
Normal file
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"_sliding_window_pattern": 6,
|
||||
"architectures": [
|
||||
"Gemma3ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attn_logit_softcapping": null,
|
||||
"bos_token_id": 2,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 1,
|
||||
"final_logit_softcapping": null,
|
||||
"head_dim": 256,
|
||||
"hidden_activation": "gelu_pytorch_tanh",
|
||||
"hidden_size": 640,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 2048,
|
||||
"layer_types": [
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"model_type": "gemma3_text",
|
||||
"num_attention_heads": 4,
|
||||
"num_hidden_layers": 18,
|
||||
"num_key_value_heads": 1,
|
||||
"pad_token_id": 0,
|
||||
"query_pre_attn_scalar": 256,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"full_attention": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_attention": {
|
||||
"rope_theta": 10000.0,
|
||||
"rope_type": "default"
|
||||
}
|
||||
},
|
||||
"sliding_window": 512,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.0.0",
|
||||
"use_bidirectional_attention": false,
|
||||
"use_cache": false,
|
||||
"vocab_size": 262144
|
||||
}
|
||||
13
checkpoint-213/generation_config.json
Normal file
13
checkpoint-213/generation_config.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"bos_token_id": 2,
|
||||
"cache_implementation": "hybrid",
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
1,
|
||||
106
|
||||
],
|
||||
"pad_token_id": 0,
|
||||
"top_k": 64,
|
||||
"top_p": 0.95,
|
||||
"transformers_version": "5.0.0"
|
||||
}
|
||||
3
checkpoint-213/model.safetensors
Normal file
3
checkpoint-213/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c4eaf9a42a809acc2e2e2e1a02042ad0dbd267ec8a17813c5a7a4e79f6d5f988
|
||||
size 536223056
|
||||
3
checkpoint-213/optimizer.pt
Normal file
3
checkpoint-213/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:7b2068fdfa062a74347042b74ff1e647b9bb9b3a0d868506e385e0eba26e216e
|
||||
size 1072591115
|
||||
3
checkpoint-213/rng_state.pth
Normal file
3
checkpoint-213/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:01f9a0f7843a37be87edd23f4e88aa93b38b95cc2c07503eeb1cf2e4632453a2
|
||||
size 14645
|
||||
3
checkpoint-213/scheduler.pt
Normal file
3
checkpoint-213/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:b7aa4a23cfb25aa3311924461ac68a6e00e95172dc91f470348c1ae92a8f6ccc
|
||||
size 1465
|
||||
3
checkpoint-213/tokenizer.json
Normal file
3
checkpoint-213/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497
|
||||
size 33384443
|
||||
24
checkpoint-213/tokenizer_config.json
Normal file
24
checkpoint-213/tokenizer_config.json
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"backend": "tokenizers",
|
||||
"boi_token": "<start_of_image>",
|
||||
"bos_token": "<bos>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eoi_token": "<end_of_image>",
|
||||
"eos_token": "<eos>",
|
||||
"image_token": "<image_soft_token>",
|
||||
"is_local": false,
|
||||
"mask_token": "<mask>",
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"model_specific_special_tokens": {
|
||||
"boi_token": "<start_of_image>",
|
||||
"eoi_token": "<end_of_image>",
|
||||
"image_token": "<image_soft_token>"
|
||||
},
|
||||
"pad_token": "<pad>",
|
||||
"padding_side": "left",
|
||||
"sp_model_kwargs": null,
|
||||
"spaces_between_special_tokens": false,
|
||||
"tokenizer_class": "GemmaTokenizer",
|
||||
"unk_token": "<unk>",
|
||||
"use_default_system_prompt": false
|
||||
}
|
||||
2197
checkpoint-213/trainer_state.json
Normal file
2197
checkpoint-213/trainer_state.json
Normal file
File diff suppressed because it is too large
Load Diff
3
checkpoint-213/training_args.bin
Normal file
3
checkpoint-213/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f426a6e6806cf7029aec0a6fb169b02e14452d298ed1e630560f4ded93439cb9
|
||||
size 5585
|
||||
47
checkpoint-284/chat_template.jinja
Normal file
47
checkpoint-284/chat_template.jinja
Normal file
@@ -0,0 +1,47 @@
|
||||
{{ bos_token }}
|
||||
{%- if messages[0]['role'] == 'system' -%}
|
||||
{%- if messages[0]['content'] is string -%}
|
||||
{%- set first_user_prefix = messages[0]['content'] + '
|
||||
|
||||
' -%}
|
||||
{%- else -%}
|
||||
{%- set first_user_prefix = messages[0]['content'][0]['text'] + '
|
||||
|
||||
' -%}
|
||||
{%- endif -%}
|
||||
{%- set loop_messages = messages[1:] -%}
|
||||
{%- else -%}
|
||||
{%- set first_user_prefix = "" -%}
|
||||
{%- set loop_messages = messages -%}
|
||||
{%- endif -%}
|
||||
{%- for message in loop_messages -%}
|
||||
{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
|
||||
{{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
|
||||
{%- endif -%}
|
||||
{%- if (message['role'] == 'assistant') -%}
|
||||
{%- set role = "model" -%}
|
||||
{%- else -%}
|
||||
{%- set role = message['role'] -%}
|
||||
{%- endif -%}
|
||||
{{ '<start_of_turn>' + role + '
|
||||
' + (first_user_prefix if loop.first else "") }}
|
||||
{%- if message['content'] is string -%}
|
||||
{{ message['content'] | trim }}
|
||||
{%- elif message['content'] is iterable -%}
|
||||
{%- for item in message['content'] -%}
|
||||
{%- if item['type'] == 'image' -%}
|
||||
{{ '<start_of_image>' }}
|
||||
{%- elif item['type'] == 'text' -%}
|
||||
{{ item['text'] | trim }}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- else -%}
|
||||
{{ raise_exception("Invalid content type") }}
|
||||
{%- endif -%}
|
||||
{{ '<end_of_turn>
|
||||
' }}
|
||||
{%- endfor -%}
|
||||
{%- if add_generation_prompt -%}
|
||||
{{'<start_of_turn>model
|
||||
'}}
|
||||
{%- endif -%}
|
||||
62
checkpoint-284/config.json
Normal file
62
checkpoint-284/config.json
Normal file
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"_sliding_window_pattern": 6,
|
||||
"architectures": [
|
||||
"Gemma3ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attn_logit_softcapping": null,
|
||||
"bos_token_id": 2,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 1,
|
||||
"final_logit_softcapping": null,
|
||||
"head_dim": 256,
|
||||
"hidden_activation": "gelu_pytorch_tanh",
|
||||
"hidden_size": 640,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 2048,
|
||||
"layer_types": [
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"model_type": "gemma3_text",
|
||||
"num_attention_heads": 4,
|
||||
"num_hidden_layers": 18,
|
||||
"num_key_value_heads": 1,
|
||||
"pad_token_id": 0,
|
||||
"query_pre_attn_scalar": 256,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"full_attention": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_attention": {
|
||||
"rope_theta": 10000.0,
|
||||
"rope_type": "default"
|
||||
}
|
||||
},
|
||||
"sliding_window": 512,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.0.0",
|
||||
"use_bidirectional_attention": false,
|
||||
"use_cache": false,
|
||||
"vocab_size": 262144
|
||||
}
|
||||
13
checkpoint-284/generation_config.json
Normal file
13
checkpoint-284/generation_config.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"bos_token_id": 2,
|
||||
"cache_implementation": "hybrid",
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
1,
|
||||
106
|
||||
],
|
||||
"pad_token_id": 0,
|
||||
"top_k": 64,
|
||||
"top_p": 0.95,
|
||||
"transformers_version": "5.0.0"
|
||||
}
|
||||
3
checkpoint-284/model.safetensors
Normal file
3
checkpoint-284/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:7d96c91306eb76ff6917e36d262afbcc696836520af2951f30d4fe02f9f50ad8
|
||||
size 536223056
|
||||
3
checkpoint-284/optimizer.pt
Normal file
3
checkpoint-284/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e1599c40ff1a4dbea243940689319509368598856b6a4085fbe267e44eeb8a59
|
||||
size 1072591115
|
||||
3
checkpoint-284/rng_state.pth
Normal file
3
checkpoint-284/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fdc37bbd2e979f041dfbbb004a5c74bab6cdda159cb18116df728588515a9ef6
|
||||
size 14645
|
||||
3
checkpoint-284/scheduler.pt
Normal file
3
checkpoint-284/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:173f9bcfe6d98de3b547cda3c500547190199c4bb0a20a7f6ab221a288667d8b
|
||||
size 1465
|
||||
3
checkpoint-284/tokenizer.json
Normal file
3
checkpoint-284/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497
|
||||
size 33384443
|
||||
24
checkpoint-284/tokenizer_config.json
Normal file
24
checkpoint-284/tokenizer_config.json
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"backend": "tokenizers",
|
||||
"boi_token": "<start_of_image>",
|
||||
"bos_token": "<bos>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eoi_token": "<end_of_image>",
|
||||
"eos_token": "<eos>",
|
||||
"image_token": "<image_soft_token>",
|
||||
"is_local": false,
|
||||
"mask_token": "<mask>",
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"model_specific_special_tokens": {
|
||||
"boi_token": "<start_of_image>",
|
||||
"eoi_token": "<end_of_image>",
|
||||
"image_token": "<image_soft_token>"
|
||||
},
|
||||
"pad_token": "<pad>",
|
||||
"padding_side": "left",
|
||||
"sp_model_kwargs": null,
|
||||
"spaces_between_special_tokens": false,
|
||||
"tokenizer_class": "GemmaTokenizer",
|
||||
"unk_token": "<unk>",
|
||||
"use_default_system_prompt": false
|
||||
}
|
||||
2918
checkpoint-284/trainer_state.json
Normal file
2918
checkpoint-284/trainer_state.json
Normal file
File diff suppressed because it is too large
Load Diff
3
checkpoint-284/training_args.bin
Normal file
3
checkpoint-284/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f426a6e6806cf7029aec0a6fb169b02e14452d298ed1e630560f4ded93439cb9
|
||||
size 5585
|
||||
47
checkpoint-71/chat_template.jinja
Normal file
47
checkpoint-71/chat_template.jinja
Normal file
@@ -0,0 +1,47 @@
|
||||
{{ bos_token }}
|
||||
{%- if messages[0]['role'] == 'system' -%}
|
||||
{%- if messages[0]['content'] is string -%}
|
||||
{%- set first_user_prefix = messages[0]['content'] + '
|
||||
|
||||
' -%}
|
||||
{%- else -%}
|
||||
{%- set first_user_prefix = messages[0]['content'][0]['text'] + '
|
||||
|
||||
' -%}
|
||||
{%- endif -%}
|
||||
{%- set loop_messages = messages[1:] -%}
|
||||
{%- else -%}
|
||||
{%- set first_user_prefix = "" -%}
|
||||
{%- set loop_messages = messages -%}
|
||||
{%- endif -%}
|
||||
{%- for message in loop_messages -%}
|
||||
{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
|
||||
{{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
|
||||
{%- endif -%}
|
||||
{%- if (message['role'] == 'assistant') -%}
|
||||
{%- set role = "model" -%}
|
||||
{%- else -%}
|
||||
{%- set role = message['role'] -%}
|
||||
{%- endif -%}
|
||||
{{ '<start_of_turn>' + role + '
|
||||
' + (first_user_prefix if loop.first else "") }}
|
||||
{%- if message['content'] is string -%}
|
||||
{{ message['content'] | trim }}
|
||||
{%- elif message['content'] is iterable -%}
|
||||
{%- for item in message['content'] -%}
|
||||
{%- if item['type'] == 'image' -%}
|
||||
{{ '<start_of_image>' }}
|
||||
{%- elif item['type'] == 'text' -%}
|
||||
{{ item['text'] | trim }}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- else -%}
|
||||
{{ raise_exception("Invalid content type") }}
|
||||
{%- endif -%}
|
||||
{{ '<end_of_turn>
|
||||
' }}
|
||||
{%- endfor -%}
|
||||
{%- if add_generation_prompt -%}
|
||||
{{'<start_of_turn>model
|
||||
'}}
|
||||
{%- endif -%}
|
||||
62
checkpoint-71/config.json
Normal file
62
checkpoint-71/config.json
Normal file
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"_sliding_window_pattern": 6,
|
||||
"architectures": [
|
||||
"Gemma3ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attn_logit_softcapping": null,
|
||||
"bos_token_id": 2,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 1,
|
||||
"final_logit_softcapping": null,
|
||||
"head_dim": 256,
|
||||
"hidden_activation": "gelu_pytorch_tanh",
|
||||
"hidden_size": 640,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 2048,
|
||||
"layer_types": [
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"model_type": "gemma3_text",
|
||||
"num_attention_heads": 4,
|
||||
"num_hidden_layers": 18,
|
||||
"num_key_value_heads": 1,
|
||||
"pad_token_id": 0,
|
||||
"query_pre_attn_scalar": 256,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"full_attention": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_attention": {
|
||||
"rope_theta": 10000.0,
|
||||
"rope_type": "default"
|
||||
}
|
||||
},
|
||||
"sliding_window": 512,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.0.0",
|
||||
"use_bidirectional_attention": false,
|
||||
"use_cache": false,
|
||||
"vocab_size": 262144
|
||||
}
|
||||
13
checkpoint-71/generation_config.json
Normal file
13
checkpoint-71/generation_config.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"bos_token_id": 2,
|
||||
"cache_implementation": "hybrid",
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
1,
|
||||
106
|
||||
],
|
||||
"pad_token_id": 0,
|
||||
"top_k": 64,
|
||||
"top_p": 0.95,
|
||||
"transformers_version": "5.0.0"
|
||||
}
|
||||
3
checkpoint-71/model.safetensors
Normal file
3
checkpoint-71/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ce662878ad6e9761f44d14c64b8ad018edf716822dba4c94e827070456a6a17b
|
||||
size 536223056
|
||||
3
checkpoint-71/optimizer.pt
Normal file
3
checkpoint-71/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3a90c0c913beec5aee54988304a77e9b75b4259d9d1d6af43ff219239b2b331c
|
||||
size 1072591115
|
||||
3
checkpoint-71/rng_state.pth
Normal file
3
checkpoint-71/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878
|
||||
size 14645
|
||||
3
checkpoint-71/scheduler.pt
Normal file
3
checkpoint-71/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3b6cac40310cacf8cbaf2f5446f5795764e3a9cd1001f02bd65f1eb2fb5fd2ed
|
||||
size 1465
|
||||
3
checkpoint-71/tokenizer.json
Normal file
3
checkpoint-71/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497
|
||||
size 33384443
|
||||
24
checkpoint-71/tokenizer_config.json
Normal file
24
checkpoint-71/tokenizer_config.json
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"backend": "tokenizers",
|
||||
"boi_token": "<start_of_image>",
|
||||
"bos_token": "<bos>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eoi_token": "<end_of_image>",
|
||||
"eos_token": "<eos>",
|
||||
"image_token": "<image_soft_token>",
|
||||
"is_local": false,
|
||||
"mask_token": "<mask>",
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"model_specific_special_tokens": {
|
||||
"boi_token": "<start_of_image>",
|
||||
"eoi_token": "<end_of_image>",
|
||||
"image_token": "<image_soft_token>"
|
||||
},
|
||||
"pad_token": "<pad>",
|
||||
"padding_side": "left",
|
||||
"sp_model_kwargs": null,
|
||||
"spaces_between_special_tokens": false,
|
||||
"tokenizer_class": "GemmaTokenizer",
|
||||
"unk_token": "<unk>",
|
||||
"use_default_system_prompt": false
|
||||
}
|
||||
755
checkpoint-71/trainer_state.json
Normal file
755
checkpoint-71/trainer_state.json
Normal file
@@ -0,0 +1,755 @@
|
||||
{
|
||||
"best_global_step": null,
|
||||
"best_metric": null,
|
||||
"best_model_checkpoint": null,
|
||||
"epoch": 1.0,
|
||||
"eval_steps": 500,
|
||||
"global_step": 71,
|
||||
"is_hyper_param_search": false,
|
||||
"is_local_process_zero": true,
|
||||
"is_world_process_zero": true,
|
||||
"log_history": [
|
||||
{
|
||||
"entropy": 1.4938980638980865,
|
||||
"epoch": 0.014084507042253521,
|
||||
"grad_norm": 89.5,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 4.519379615783691,
|
||||
"mean_token_accuracy": 0.3811451345682144,
|
||||
"num_tokens": 2953.0,
|
||||
"step": 1
|
||||
},
|
||||
{
|
||||
"entropy": 3.6362282037734985,
|
||||
"epoch": 0.028169014084507043,
|
||||
"grad_norm": 29.125,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 3.7604172229766846,
|
||||
"mean_token_accuracy": 0.39103876799345016,
|
||||
"num_tokens": 5196.0,
|
||||
"step": 2
|
||||
},
|
||||
{
|
||||
"entropy": 3.796808362007141,
|
||||
"epoch": 0.04225352112676056,
|
||||
"grad_norm": 18.75,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 3.3663859367370605,
|
||||
"mean_token_accuracy": 0.42154909670352936,
|
||||
"num_tokens": 7537.0,
|
||||
"step": 3
|
||||
},
|
||||
{
|
||||
"entropy": 3.071028172969818,
|
||||
"epoch": 0.056338028169014086,
|
||||
"grad_norm": 14.5625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 3.1916425228118896,
|
||||
"mean_token_accuracy": 0.4548730403184891,
|
||||
"num_tokens": 9584.0,
|
||||
"step": 4
|
||||
},
|
||||
{
|
||||
"entropy": 2.7734431624412537,
|
||||
"epoch": 0.07042253521126761,
|
||||
"grad_norm": 15.0,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.661928653717041,
|
||||
"mean_token_accuracy": 0.5473615825176239,
|
||||
"num_tokens": 11344.0,
|
||||
"step": 5
|
||||
},
|
||||
{
|
||||
"entropy": 2.4827571511268616,
|
||||
"epoch": 0.08450704225352113,
|
||||
"grad_norm": 13.6875,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.691732168197632,
|
||||
"mean_token_accuracy": 0.5095682442188263,
|
||||
"num_tokens": 13289.0,
|
||||
"step": 6
|
||||
},
|
||||
{
|
||||
"entropy": 2.6064285039901733,
|
||||
"epoch": 0.09859154929577464,
|
||||
"grad_norm": 12.625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.7455577850341797,
|
||||
"mean_token_accuracy": 0.5099757686257362,
|
||||
"num_tokens": 15664.0,
|
||||
"step": 7
|
||||
},
|
||||
{
|
||||
"entropy": 2.4781495332717896,
|
||||
"epoch": 0.11267605633802817,
|
||||
"grad_norm": 12.75,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.5589702129364014,
|
||||
"mean_token_accuracy": 0.5618840456008911,
|
||||
"num_tokens": 17331.0,
|
||||
"step": 8
|
||||
},
|
||||
{
|
||||
"entropy": 2.512305438518524,
|
||||
"epoch": 0.1267605633802817,
|
||||
"grad_norm": 10.1875,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.5893146991729736,
|
||||
"mean_token_accuracy": 0.5188388600945473,
|
||||
"num_tokens": 19724.0,
|
||||
"step": 9
|
||||
},
|
||||
{
|
||||
"entropy": 2.187403589487076,
|
||||
"epoch": 0.14084507042253522,
|
||||
"grad_norm": 11.8125,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.4727907180786133,
|
||||
"mean_token_accuracy": 0.6085146963596344,
|
||||
"num_tokens": 21334.0,
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"entropy": 2.8956857919692993,
|
||||
"epoch": 0.15492957746478872,
|
||||
"grad_norm": 10.1875,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.8553805351257324,
|
||||
"mean_token_accuracy": 0.4754949361085892,
|
||||
"num_tokens": 23770.0,
|
||||
"step": 11
|
||||
},
|
||||
{
|
||||
"entropy": 2.4213987588882446,
|
||||
"epoch": 0.16901408450704225,
|
||||
"grad_norm": 9.75,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.4139676094055176,
|
||||
"mean_token_accuracy": 0.5475910231471062,
|
||||
"num_tokens": 26233.0,
|
||||
"step": 12
|
||||
},
|
||||
{
|
||||
"entropy": 2.668797016143799,
|
||||
"epoch": 0.18309859154929578,
|
||||
"grad_norm": 12.25,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.769174337387085,
|
||||
"mean_token_accuracy": 0.5351328551769257,
|
||||
"num_tokens": 27692.0,
|
||||
"step": 13
|
||||
},
|
||||
{
|
||||
"entropy": 2.544685661792755,
|
||||
"epoch": 0.19718309859154928,
|
||||
"grad_norm": 9.5625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.4870781898498535,
|
||||
"mean_token_accuracy": 0.514883853495121,
|
||||
"num_tokens": 30217.0,
|
||||
"step": 14
|
||||
},
|
||||
{
|
||||
"entropy": 2.5978153944015503,
|
||||
"epoch": 0.2112676056338028,
|
||||
"grad_norm": 10.875,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.639450788497925,
|
||||
"mean_token_accuracy": 0.5468774288892746,
|
||||
"num_tokens": 31969.0,
|
||||
"step": 15
|
||||
},
|
||||
{
|
||||
"entropy": 2.452792704105377,
|
||||
"epoch": 0.22535211267605634,
|
||||
"grad_norm": 11.5,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.5285415649414062,
|
||||
"mean_token_accuracy": 0.5533104464411736,
|
||||
"num_tokens": 34160.0,
|
||||
"step": 16
|
||||
},
|
||||
{
|
||||
"entropy": 2.234907388687134,
|
||||
"epoch": 0.23943661971830985,
|
||||
"grad_norm": 12.0625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.2553577423095703,
|
||||
"mean_token_accuracy": 0.5958019495010376,
|
||||
"num_tokens": 35899.0,
|
||||
"step": 17
|
||||
},
|
||||
{
|
||||
"entropy": 2.2781192660331726,
|
||||
"epoch": 0.2535211267605634,
|
||||
"grad_norm": 10.8125,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.4401230812072754,
|
||||
"mean_token_accuracy": 0.5632342845201492,
|
||||
"num_tokens": 37693.0,
|
||||
"step": 18
|
||||
},
|
||||
{
|
||||
"entropy": 2.292062520980835,
|
||||
"epoch": 0.2676056338028169,
|
||||
"grad_norm": 10.5,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.53348445892334,
|
||||
"mean_token_accuracy": 0.5441952645778656,
|
||||
"num_tokens": 39793.0,
|
||||
"step": 19
|
||||
},
|
||||
{
|
||||
"entropy": 2.5250023007392883,
|
||||
"epoch": 0.28169014084507044,
|
||||
"grad_norm": 13.5,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.7173304557800293,
|
||||
"mean_token_accuracy": 0.4843936040997505,
|
||||
"num_tokens": 42035.0,
|
||||
"step": 20
|
||||
},
|
||||
{
|
||||
"entropy": 2.3936269879341125,
|
||||
"epoch": 0.29577464788732394,
|
||||
"grad_norm": 10.5625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.397752046585083,
|
||||
"mean_token_accuracy": 0.5632070526480675,
|
||||
"num_tokens": 43606.0,
|
||||
"step": 21
|
||||
},
|
||||
{
|
||||
"entropy": 2.7215799689292908,
|
||||
"epoch": 0.30985915492957744,
|
||||
"grad_norm": 9.375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.7266643047332764,
|
||||
"mean_token_accuracy": 0.4814189076423645,
|
||||
"num_tokens": 46160.0,
|
||||
"step": 22
|
||||
},
|
||||
{
|
||||
"entropy": 2.4695653319358826,
|
||||
"epoch": 0.323943661971831,
|
||||
"grad_norm": 8.4375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.3790149688720703,
|
||||
"mean_token_accuracy": 0.5413414835929871,
|
||||
"num_tokens": 48749.0,
|
||||
"step": 23
|
||||
},
|
||||
{
|
||||
"entropy": 2.3166028261184692,
|
||||
"epoch": 0.3380281690140845,
|
||||
"grad_norm": 8.9375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.274803638458252,
|
||||
"mean_token_accuracy": 0.5993590205907822,
|
||||
"num_tokens": 50770.0,
|
||||
"step": 24
|
||||
},
|
||||
{
|
||||
"entropy": 2.285377472639084,
|
||||
"epoch": 0.352112676056338,
|
||||
"grad_norm": 8.75,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.2308878898620605,
|
||||
"mean_token_accuracy": 0.5828141123056412,
|
||||
"num_tokens": 53004.0,
|
||||
"step": 25
|
||||
},
|
||||
{
|
||||
"entropy": 2.1157060265541077,
|
||||
"epoch": 0.36619718309859156,
|
||||
"grad_norm": 8.375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.1033847332000732,
|
||||
"mean_token_accuracy": 0.5867631435394287,
|
||||
"num_tokens": 55108.0,
|
||||
"step": 26
|
||||
},
|
||||
{
|
||||
"entropy": 2.5209829807281494,
|
||||
"epoch": 0.38028169014084506,
|
||||
"grad_norm": 8.9375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.3607499599456787,
|
||||
"mean_token_accuracy": 0.5426046550273895,
|
||||
"num_tokens": 57080.0,
|
||||
"step": 27
|
||||
},
|
||||
{
|
||||
"entropy": 1.974183738231659,
|
||||
"epoch": 0.39436619718309857,
|
||||
"grad_norm": 7.4375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 1.9418365955352783,
|
||||
"mean_token_accuracy": 0.6255736351013184,
|
||||
"num_tokens": 59237.0,
|
||||
"step": 28
|
||||
},
|
||||
{
|
||||
"entropy": 2.252374231815338,
|
||||
"epoch": 0.4084507042253521,
|
||||
"grad_norm": 9.875,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.228358507156372,
|
||||
"mean_token_accuracy": 0.5917280316352844,
|
||||
"num_tokens": 60850.0,
|
||||
"step": 29
|
||||
},
|
||||
{
|
||||
"entropy": 2.327530801296234,
|
||||
"epoch": 0.4225352112676056,
|
||||
"grad_norm": 9.5625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.3329389095306396,
|
||||
"mean_token_accuracy": 0.5718153715133667,
|
||||
"num_tokens": 62734.0,
|
||||
"step": 30
|
||||
},
|
||||
{
|
||||
"entropy": 2.2620502710342407,
|
||||
"epoch": 0.43661971830985913,
|
||||
"grad_norm": 9.375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.1953892707824707,
|
||||
"mean_token_accuracy": 0.5675001591444016,
|
||||
"num_tokens": 64637.0,
|
||||
"step": 31
|
||||
},
|
||||
{
|
||||
"entropy": 2.445267140865326,
|
||||
"epoch": 0.4507042253521127,
|
||||
"grad_norm": 9.0625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.501152276992798,
|
||||
"mean_token_accuracy": 0.5355894565582275,
|
||||
"num_tokens": 66880.0,
|
||||
"step": 32
|
||||
},
|
||||
{
|
||||
"entropy": 2.3873718976974487,
|
||||
"epoch": 0.4647887323943662,
|
||||
"grad_norm": 9.4375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.3468880653381348,
|
||||
"mean_token_accuracy": 0.5372899621725082,
|
||||
"num_tokens": 68623.0,
|
||||
"step": 33
|
||||
},
|
||||
{
|
||||
"entropy": 2.0545941591262817,
|
||||
"epoch": 0.4788732394366197,
|
||||
"grad_norm": 8.9375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.1947593688964844,
|
||||
"mean_token_accuracy": 0.6112199202179909,
|
||||
"num_tokens": 70769.0,
|
||||
"step": 34
|
||||
},
|
||||
{
|
||||
"entropy": 2.350032150745392,
|
||||
"epoch": 0.49295774647887325,
|
||||
"grad_norm": 8.0625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.418266773223877,
|
||||
"mean_token_accuracy": 0.5104246437549591,
|
||||
"num_tokens": 73653.0,
|
||||
"step": 35
|
||||
},
|
||||
{
|
||||
"entropy": 2.237062931060791,
|
||||
"epoch": 0.5070422535211268,
|
||||
"grad_norm": 10.75,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.4755356311798096,
|
||||
"mean_token_accuracy": 0.5747656747698784,
|
||||
"num_tokens": 75524.0,
|
||||
"step": 36
|
||||
},
|
||||
{
|
||||
"entropy": 2.0994813442230225,
|
||||
"epoch": 0.5211267605633803,
|
||||
"grad_norm": 8.9375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.044315814971924,
|
||||
"mean_token_accuracy": 0.6056438237428665,
|
||||
"num_tokens": 77246.0,
|
||||
"step": 37
|
||||
},
|
||||
{
|
||||
"entropy": 2.5556156635284424,
|
||||
"epoch": 0.5352112676056338,
|
||||
"grad_norm": 8.25,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.446904182434082,
|
||||
"mean_token_accuracy": 0.5177429020404816,
|
||||
"num_tokens": 79970.0,
|
||||
"step": 38
|
||||
},
|
||||
{
|
||||
"entropy": 2.337540030479431,
|
||||
"epoch": 0.5492957746478874,
|
||||
"grad_norm": 8.5625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.216331720352173,
|
||||
"mean_token_accuracy": 0.58082115650177,
|
||||
"num_tokens": 81876.0,
|
||||
"step": 39
|
||||
},
|
||||
{
|
||||
"entropy": 2.043498247861862,
|
||||
"epoch": 0.5633802816901409,
|
||||
"grad_norm": 8.0625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 1.9643354415893555,
|
||||
"mean_token_accuracy": 0.607293888926506,
|
||||
"num_tokens": 84298.0,
|
||||
"step": 40
|
||||
},
|
||||
{
|
||||
"entropy": 2.4852016270160675,
|
||||
"epoch": 0.5774647887323944,
|
||||
"grad_norm": 8.75,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.571437358856201,
|
||||
"mean_token_accuracy": 0.5233406946063042,
|
||||
"num_tokens": 86291.0,
|
||||
"step": 41
|
||||
},
|
||||
{
|
||||
"entropy": 2.5456899404525757,
|
||||
"epoch": 0.5915492957746479,
|
||||
"grad_norm": 9.125,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.4434070587158203,
|
||||
"mean_token_accuracy": 0.5472028106451035,
|
||||
"num_tokens": 88194.0,
|
||||
"step": 42
|
||||
},
|
||||
{
|
||||
"entropy": 2.2191686630249023,
|
||||
"epoch": 0.6056338028169014,
|
||||
"grad_norm": 9.0625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.225029468536377,
|
||||
"mean_token_accuracy": 0.6006355881690979,
|
||||
"num_tokens": 89922.0,
|
||||
"step": 43
|
||||
},
|
||||
{
|
||||
"entropy": 2.2295450568199158,
|
||||
"epoch": 0.6197183098591549,
|
||||
"grad_norm": 8.8125,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.4000232219696045,
|
||||
"mean_token_accuracy": 0.580116793513298,
|
||||
"num_tokens": 92171.0,
|
||||
"step": 44
|
||||
},
|
||||
{
|
||||
"entropy": 2.2142042219638824,
|
||||
"epoch": 0.6338028169014085,
|
||||
"grad_norm": 8.3125,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.2619423866271973,
|
||||
"mean_token_accuracy": 0.5818447023630142,
|
||||
"num_tokens": 94362.0,
|
||||
"step": 45
|
||||
},
|
||||
{
|
||||
"entropy": 2.3695861101150513,
|
||||
"epoch": 0.647887323943662,
|
||||
"grad_norm": 7.21875,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.4540324211120605,
|
||||
"mean_token_accuracy": 0.5298221409320831,
|
||||
"num_tokens": 96559.0,
|
||||
"step": 46
|
||||
},
|
||||
{
|
||||
"entropy": 1.9680090248584747,
|
||||
"epoch": 0.6619718309859155,
|
||||
"grad_norm": 8.9375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 1.9608612060546875,
|
||||
"mean_token_accuracy": 0.6151116490364075,
|
||||
"num_tokens": 99018.0,
|
||||
"step": 47
|
||||
},
|
||||
{
|
||||
"entropy": 2.449136793613434,
|
||||
"epoch": 0.676056338028169,
|
||||
"grad_norm": 9.375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.364119529724121,
|
||||
"mean_token_accuracy": 0.5657317340373993,
|
||||
"num_tokens": 100885.0,
|
||||
"step": 48
|
||||
},
|
||||
{
|
||||
"entropy": 2.109535813331604,
|
||||
"epoch": 0.6901408450704225,
|
||||
"grad_norm": 6.875,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.1192779541015625,
|
||||
"mean_token_accuracy": 0.5970411896705627,
|
||||
"num_tokens": 103357.0,
|
||||
"step": 49
|
||||
},
|
||||
{
|
||||
"entropy": 2.3080650568008423,
|
||||
"epoch": 0.704225352112676,
|
||||
"grad_norm": 8.0625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.351897716522217,
|
||||
"mean_token_accuracy": 0.5307685062289238,
|
||||
"num_tokens": 105870.0,
|
||||
"step": 50
|
||||
},
|
||||
{
|
||||
"entropy": 2.1463415920734406,
|
||||
"epoch": 0.7183098591549296,
|
||||
"grad_norm": 7.5,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.2693376541137695,
|
||||
"mean_token_accuracy": 0.5934874191880226,
|
||||
"num_tokens": 107995.0,
|
||||
"step": 51
|
||||
},
|
||||
{
|
||||
"entropy": 2.1181687712669373,
|
||||
"epoch": 0.7323943661971831,
|
||||
"grad_norm": 8.4375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.1629841327667236,
|
||||
"mean_token_accuracy": 0.6063478291034698,
|
||||
"num_tokens": 110358.0,
|
||||
"step": 52
|
||||
},
|
||||
{
|
||||
"entropy": 2.0169546008110046,
|
||||
"epoch": 0.7464788732394366,
|
||||
"grad_norm": 7.78125,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 1.817868709564209,
|
||||
"mean_token_accuracy": 0.6273332387208939,
|
||||
"num_tokens": 112267.0,
|
||||
"step": 53
|
||||
},
|
||||
{
|
||||
"entropy": 2.189443975687027,
|
||||
"epoch": 0.7605633802816901,
|
||||
"grad_norm": 9.8125,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.3292086124420166,
|
||||
"mean_token_accuracy": 0.6041209101676941,
|
||||
"num_tokens": 113899.0,
|
||||
"step": 54
|
||||
},
|
||||
{
|
||||
"entropy": 2.1946000456809998,
|
||||
"epoch": 0.7746478873239436,
|
||||
"grad_norm": 9.3125,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.2774949073791504,
|
||||
"mean_token_accuracy": 0.5620942413806915,
|
||||
"num_tokens": 116015.0,
|
||||
"step": 55
|
||||
},
|
||||
{
|
||||
"entropy": 2.1172711849212646,
|
||||
"epoch": 0.7887323943661971,
|
||||
"grad_norm": 7.90625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.1236391067504883,
|
||||
"mean_token_accuracy": 0.5911316424608231,
|
||||
"num_tokens": 118044.0,
|
||||
"step": 56
|
||||
},
|
||||
{
|
||||
"entropy": 2.144031971693039,
|
||||
"epoch": 0.8028169014084507,
|
||||
"grad_norm": 8.8125,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.187140703201294,
|
||||
"mean_token_accuracy": 0.5624879449605942,
|
||||
"num_tokens": 120180.0,
|
||||
"step": 57
|
||||
},
|
||||
{
|
||||
"entropy": 2.181178390979767,
|
||||
"epoch": 0.8169014084507042,
|
||||
"grad_norm": 10.3125,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.113279104232788,
|
||||
"mean_token_accuracy": 0.6004882454872131,
|
||||
"num_tokens": 121676.0,
|
||||
"step": 58
|
||||
},
|
||||
{
|
||||
"entropy": 2.314671218395233,
|
||||
"epoch": 0.8309859154929577,
|
||||
"grad_norm": 9.5625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.331024169921875,
|
||||
"mean_token_accuracy": 0.5824068263173103,
|
||||
"num_tokens": 123161.0,
|
||||
"step": 59
|
||||
},
|
||||
{
|
||||
"entropy": 2.0048855245113373,
|
||||
"epoch": 0.8450704225352113,
|
||||
"grad_norm": 9.375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.070265054702759,
|
||||
"mean_token_accuracy": 0.6141902804374695,
|
||||
"num_tokens": 125016.0,
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"entropy": 1.966452568769455,
|
||||
"epoch": 0.8591549295774648,
|
||||
"grad_norm": 7.96875,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 1.9488521814346313,
|
||||
"mean_token_accuracy": 0.5854089111089706,
|
||||
"num_tokens": 127349.0,
|
||||
"step": 61
|
||||
},
|
||||
{
|
||||
"entropy": 2.1519170999526978,
|
||||
"epoch": 0.8732394366197183,
|
||||
"grad_norm": 7.5625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.1743807792663574,
|
||||
"mean_token_accuracy": 0.5636171847581863,
|
||||
"num_tokens": 129742.0,
|
||||
"step": 62
|
||||
},
|
||||
{
|
||||
"entropy": 2.7546687722206116,
|
||||
"epoch": 0.8873239436619719,
|
||||
"grad_norm": 7.96875,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.7343056201934814,
|
||||
"mean_token_accuracy": 0.496803916990757,
|
||||
"num_tokens": 131743.0,
|
||||
"step": 63
|
||||
},
|
||||
{
|
||||
"entropy": 2.421773910522461,
|
||||
"epoch": 0.9014084507042254,
|
||||
"grad_norm": 7.96875,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.6149866580963135,
|
||||
"mean_token_accuracy": 0.5249136164784431,
|
||||
"num_tokens": 134002.0,
|
||||
"step": 64
|
||||
},
|
||||
{
|
||||
"entropy": 2.0106727480888367,
|
||||
"epoch": 0.9154929577464789,
|
||||
"grad_norm": 8.4375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 1.940914273262024,
|
||||
"mean_token_accuracy": 0.5855439230799675,
|
||||
"num_tokens": 136120.0,
|
||||
"step": 65
|
||||
},
|
||||
{
|
||||
"entropy": 2.1143648624420166,
|
||||
"epoch": 0.9295774647887324,
|
||||
"grad_norm": 8.375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.2469482421875,
|
||||
"mean_token_accuracy": 0.6145937740802765,
|
||||
"num_tokens": 137997.0,
|
||||
"step": 66
|
||||
},
|
||||
{
|
||||
"entropy": 2.1894670128822327,
|
||||
"epoch": 0.9436619718309859,
|
||||
"grad_norm": 8.25,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.20756459236145,
|
||||
"mean_token_accuracy": 0.5872255340218544,
|
||||
"num_tokens": 139904.0,
|
||||
"step": 67
|
||||
},
|
||||
{
|
||||
"entropy": 1.75501748919487,
|
||||
"epoch": 0.9577464788732394,
|
||||
"grad_norm": 9.3125,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 1.9068934917449951,
|
||||
"mean_token_accuracy": 0.6642919778823853,
|
||||
"num_tokens": 141166.0,
|
||||
"step": 68
|
||||
},
|
||||
{
|
||||
"entropy": 2.204477220773697,
|
||||
"epoch": 0.971830985915493,
|
||||
"grad_norm": 7.875,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.286216974258423,
|
||||
"mean_token_accuracy": 0.5354761853814125,
|
||||
"num_tokens": 143738.0,
|
||||
"step": 69
|
||||
},
|
||||
{
|
||||
"entropy": 2.1219914853572845,
|
||||
"epoch": 0.9859154929577465,
|
||||
"grad_norm": 8.625,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 2.274585723876953,
|
||||
"mean_token_accuracy": 0.609759122133255,
|
||||
"num_tokens": 145439.0,
|
||||
"step": 70
|
||||
},
|
||||
{
|
||||
"entropy": 1.9260511100292206,
|
||||
"epoch": 1.0,
|
||||
"grad_norm": 8.375,
|
||||
"learning_rate": 5e-05,
|
||||
"loss": 1.8763841390609741,
|
||||
"mean_token_accuracy": 0.5979989916086197,
|
||||
"num_tokens": 147650.0,
|
||||
"step": 71
|
||||
},
|
||||
{
|
||||
"epoch": 1.0,
|
||||
"eval_entropy": 2.1684498711371085,
|
||||
"eval_loss": 2.284580707550049,
|
||||
"eval_mean_token_accuracy": 0.5757196218195096,
|
||||
"eval_num_tokens": 147650.0,
|
||||
"eval_runtime": 18.8201,
|
||||
"eval_samples_per_second": 15.09,
|
||||
"eval_steps_per_second": 3.773,
|
||||
"step": 71
|
||||
}
|
||||
],
|
||||
"logging_steps": 1,
|
||||
"max_steps": 284,
|
||||
"num_input_tokens_seen": 0,
|
||||
"num_train_epochs": 4,
|
||||
"save_steps": 500,
|
||||
"stateful_callbacks": {
|
||||
"TrainerControl": {
|
||||
"args": {
|
||||
"should_epoch_stop": false,
|
||||
"should_evaluate": false,
|
||||
"should_log": false,
|
||||
"should_save": true,
|
||||
"should_training_stop": false
|
||||
},
|
||||
"attributes": {}
|
||||
}
|
||||
},
|
||||
"total_flos": 153041317847040.0,
|
||||
"train_batch_size": 4,
|
||||
"trial_name": null,
|
||||
"trial_params": null
|
||||
}
|
||||
3
checkpoint-71/training_args.bin
Normal file
3
checkpoint-71/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f426a6e6806cf7029aec0a6fb169b02e14452d298ed1e630560f4ded93439cb9
|
||||
size 5585
|
||||
62
config.json
Normal file
62
config.json
Normal file
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"_sliding_window_pattern": 6,
|
||||
"architectures": [
|
||||
"Gemma3ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attn_logit_softcapping": null,
|
||||
"bos_token_id": 2,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 1,
|
||||
"final_logit_softcapping": null,
|
||||
"head_dim": 256,
|
||||
"hidden_activation": "gelu_pytorch_tanh",
|
||||
"hidden_size": 640,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 2048,
|
||||
"layer_types": [
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"sliding_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"model_type": "gemma3_text",
|
||||
"num_attention_heads": 4,
|
||||
"num_hidden_layers": 18,
|
||||
"num_key_value_heads": 1,
|
||||
"pad_token_id": 0,
|
||||
"query_pre_attn_scalar": 256,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_parameters": {
|
||||
"full_attention": {
|
||||
"rope_theta": 1000000.0,
|
||||
"rope_type": "default"
|
||||
},
|
||||
"sliding_attention": {
|
||||
"rope_theta": 10000.0,
|
||||
"rope_type": "default"
|
||||
}
|
||||
},
|
||||
"sliding_window": 512,
|
||||
"tie_word_embeddings": true,
|
||||
"transformers_version": "5.0.0",
|
||||
"use_bidirectional_attention": false,
|
||||
"use_cache": false,
|
||||
"vocab_size": 262144
|
||||
}
|
||||
13
generation_config.json
Normal file
13
generation_config.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"bos_token_id": 2,
|
||||
"cache_implementation": "hybrid",
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
1,
|
||||
106
|
||||
],
|
||||
"pad_token_id": 0,
|
||||
"top_k": 64,
|
||||
"top_p": 0.95,
|
||||
"transformers_version": "5.0.0"
|
||||
}
|
||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:7d96c91306eb76ff6917e36d262afbcc696836520af2951f30d4fe02f9f50ad8
|
||||
size 536223056
|
||||
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a74aefb1dc1340a25f29ab8370384b9ed24b2d921d7749ece7bbcfcfdf00d497
|
||||
size 33384443
|
||||
24
tokenizer_config.json
Normal file
24
tokenizer_config.json
Normal file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"backend": "tokenizers",
|
||||
"boi_token": "<start_of_image>",
|
||||
"bos_token": "<bos>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eoi_token": "<end_of_image>",
|
||||
"eos_token": "<eos>",
|
||||
"image_token": "<image_soft_token>",
|
||||
"is_local": false,
|
||||
"mask_token": "<mask>",
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"model_specific_special_tokens": {
|
||||
"boi_token": "<start_of_image>",
|
||||
"eoi_token": "<end_of_image>",
|
||||
"image_token": "<image_soft_token>"
|
||||
},
|
||||
"pad_token": "<pad>",
|
||||
"padding_side": "left",
|
||||
"sp_model_kwargs": null,
|
||||
"spaces_between_special_tokens": false,
|
||||
"tokenizer_class": "GemmaTokenizer",
|
||||
"unk_token": "<unk>",
|
||||
"use_default_system_prompt": false
|
||||
}
|
||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f426a6e6806cf7029aec0a6fb169b02e14452d298ed1e630560f4ded93439cb9
|
||||
size 5585
|
||||
Reference in New Issue
Block a user