初始化项目，由ModelHub XC社区提供模型

Model: misterJB/tata-field-432hz
Source: Original Platform

.gitattributes

@@ -0,0 +1,37 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+last-checkpoint/tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,59 @@
+---
+base_model: meta-llama/Llama-3.2-3B-Instruct
+library_name: transformers
+model_name: tata-field-432hz
+tags:
+- generated_from_trainer
+- hf_jobs
+- trl
+- sft
+licence: license
+---
+
+# Model Card for tata-field-432hz
+
+This model is a fine-tuned version of [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="misterJB/tata-field-432hz", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 0.29.1
+- Transformers: 5.4.0
+- Pytorch: 2.8.0
+- Datasets: 4.8.4
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128009,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": 128009,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.4.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128009,
+    128001,
+    128008,
+    128009
+  ],
+  "pad_token_id": 128009,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.4.0"
+}

last-checkpoint/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

last-checkpoint/config.json

@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128009,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pad_token_id": 128009,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.3.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}

last-checkpoint/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128009,
+    128001,
+    128008,
+    128009
+  ],
+  "pad_token_id": 128009,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.3.0"
+}

last-checkpoint/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57c4831d4e47f3c33f5f8969a02983374946ab3e5f383e907aca585a86e5b94d
+size 6425529112

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4159a3207a568baa044ce95dd31d1026fc5b9d5c7ec0e1a1965827fde31d1cab
+size 12851224679

last-checkpoint/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878
+size 14645

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ab5f3bc9f12c23cdfc964a4b7a0357b0f5ab599c9285206de2dfd5600d45e1d
+size 1465

last-checkpoint/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

last-checkpoint/trainer_state.json

@@ -0,0 +1,534 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.953125,
+  "eval_steps": 500,
+  "global_step": 2500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.351678059399128,
+      "epoch": 0.0390625,
+      "grad_norm": 1.6015625,
+      "learning_rate": 3.828125000000001e-06,
+      "loss": 1.394322509765625,
+      "mean_token_accuracy": 0.7548403647542,
+      "num_tokens": 257685.0,
+      "step": 50
+    },
+    {
+      "entropy": 0.47313837975263595,
+      "epoch": 0.078125,
+      "grad_norm": 0.91015625,
+      "learning_rate": 7.734375e-06,
+      "loss": 0.4260553359985352,
+      "mean_token_accuracy": 0.9251225611567497,
+      "num_tokens": 513846.0,
+      "step": 100
+    },
+    {
+      "entropy": 0.22530030721798538,
+      "epoch": 0.1171875,
+      "grad_norm": 0.921875,
+      "learning_rate": 1.1640625000000002e-05,
+      "loss": 0.18161891937255858,
+      "mean_token_accuracy": 0.9637172383069992,
+      "num_tokens": 766003.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.14051863566040992,
+      "epoch": 0.15625,
+      "grad_norm": 1.6484375,
+      "learning_rate": 1.5546875e-05,
+      "loss": 0.1136919116973877,
+      "mean_token_accuracy": 0.9747392472624778,
+      "num_tokens": 1024512.0,
+      "step": 200
+    },
+    {
+      "entropy": 0.09577633743174374,
+      "epoch": 0.1953125,
+      "grad_norm": 0.71484375,
+      "learning_rate": 1.9453125e-05,
+      "loss": 0.0734261655807495,
+      "mean_token_accuracy": 0.9820188581943512,
+      "num_tokens": 1280614.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.08148466867394745,
+      "epoch": 0.234375,
+      "grad_norm": 0.3828125,
+      "learning_rate": 1.9626736111111114e-05,
+      "loss": 0.062112469673156735,
+      "mean_token_accuracy": 0.9844269120693206,
+      "num_tokens": 1533690.0,
+      "step": 300
+    },
+    {
+      "entropy": 0.06672279690392316,
+      "epoch": 0.2734375,
+      "grad_norm": 0.455078125,
+      "learning_rate": 1.9192708333333335e-05,
+      "loss": 0.05034114837646484,
+      "mean_token_accuracy": 0.986752623617649,
+      "num_tokens": 1783169.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.051021190043538805,
+      "epoch": 0.3125,
+      "grad_norm": 0.49609375,
+      "learning_rate": 1.8758680555555557e-05,
+      "loss": 0.03769558668136597,
+      "mean_token_accuracy": 0.9895561364293098,
+      "num_tokens": 2039809.0,
+      "step": 400
+    },
+    {
+      "entropy": 0.04628240401856601,
+      "epoch": 0.3515625,
+      "grad_norm": 0.1962890625,
+      "learning_rate": 1.8324652777777778e-05,
+      "loss": 0.034056272506713864,
+      "mean_token_accuracy": 0.9902530950307846,
+      "num_tokens": 2296699.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.041101934388279915,
+      "epoch": 0.390625,
+      "grad_norm": 0.498046875,
+      "learning_rate": 1.7890625000000003e-05,
+      "loss": 0.030158956050872803,
+      "mean_token_accuracy": 0.9908681440353394,
+      "num_tokens": 2551050.0,
+      "step": 500
+    },
+    {
+      "entropy": 0.046159422053024174,
+      "epoch": 0.4296875,
+      "grad_norm": 0.2080078125,
+      "learning_rate": 1.7456597222222224e-05,
+      "loss": 0.03463820457458496,
+      "mean_token_accuracy": 0.9898904532194137,
+      "num_tokens": 2802698.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.03955025893636048,
+      "epoch": 0.46875,
+      "grad_norm": 0.31640625,
+      "learning_rate": 1.7022569444444446e-05,
+      "loss": 0.029909002780914306,
+      "mean_token_accuracy": 0.9910529521107674,
+      "num_tokens": 3060791.0,
+      "step": 600
+    },
+    {
+      "entropy": 0.040935935722664,
+      "epoch": 0.5078125,
+      "grad_norm": 0.28125,
+      "learning_rate": 1.6588541666666667e-05,
+      "loss": 0.030300111770629884,
+      "mean_token_accuracy": 0.9909433552622795,
+      "num_tokens": 3313602.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.03562260726466775,
+      "epoch": 0.546875,
+      "grad_norm": 0.2119140625,
+      "learning_rate": 1.615451388888889e-05,
+      "loss": 0.025539636611938477,
+      "mean_token_accuracy": 0.9918221846222878,
+      "num_tokens": 3572334.0,
+      "step": 700
+    },
+    {
+      "entropy": 0.035833339411765336,
+      "epoch": 0.5859375,
+      "grad_norm": 0.1669921875,
+      "learning_rate": 1.5720486111111114e-05,
+      "loss": 0.026018803119659425,
+      "mean_token_accuracy": 0.9919026476144791,
+      "num_tokens": 3822475.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.03438482533209026,
+      "epoch": 0.625,
+      "grad_norm": 0.318359375,
+      "learning_rate": 1.5286458333333335e-05,
+      "loss": 0.025241999626159667,
+      "mean_token_accuracy": 0.9919598492980003,
+      "num_tokens": 4077791.0,
+      "step": 800
+    },
+    {
+      "entropy": 0.03701488464139402,
+      "epoch": 0.6640625,
+      "grad_norm": 0.162109375,
+      "learning_rate": 1.4852430555555556e-05,
+      "loss": 0.0267392897605896,
+      "mean_token_accuracy": 0.9916711059212685,
+      "num_tokens": 4327537.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.03509442439302802,
+      "epoch": 0.703125,
+      "grad_norm": 0.1943359375,
+      "learning_rate": 1.4418402777777778e-05,
+      "loss": 0.025892121791839598,
+      "mean_token_accuracy": 0.9916690769791603,
+      "num_tokens": 4582391.0,
+      "step": 900
+    },
+    {
+      "entropy": 0.032335253246128556,
+      "epoch": 0.7421875,
+      "grad_norm": 0.259765625,
+      "learning_rate": 1.3984375000000001e-05,
+      "loss": 0.023846192359924315,
+      "mean_token_accuracy": 0.9924429550766944,
+      "num_tokens": 4842265.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.032289591124281286,
+      "epoch": 0.78125,
+      "grad_norm": 0.248046875,
+      "learning_rate": 1.3550347222222224e-05,
+      "loss": 0.023617899417877196,
+      "mean_token_accuracy": 0.9924442365765571,
+      "num_tokens": 5099714.0,
+      "step": 1000
+    },
+    {
+      "entropy": 0.032110756486654284,
+      "epoch": 0.8203125,
+      "grad_norm": 0.171875,
+      "learning_rate": 1.3116319444444446e-05,
+      "loss": 0.023927602767944336,
+      "mean_token_accuracy": 0.992151814699173,
+      "num_tokens": 5353934.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.03357607708312571,
+      "epoch": 0.859375,
+      "grad_norm": 0.220703125,
+      "learning_rate": 1.2682291666666669e-05,
+      "loss": 0.024996912479400633,
+      "mean_token_accuracy": 0.9920313712954522,
+      "num_tokens": 5610229.0,
+      "step": 1100
+    },
+    {
+      "entropy": 0.03356592872180045,
+      "epoch": 0.8984375,
+      "grad_norm": 0.203125,
+      "learning_rate": 1.2248263888888889e-05,
+      "loss": 0.025175034999847412,
+      "mean_token_accuracy": 0.9921249234676361,
+      "num_tokens": 5862791.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.031079287379980086,
+      "epoch": 0.9375,
+      "grad_norm": 0.1318359375,
+      "learning_rate": 1.1814236111111112e-05,
+      "loss": 0.022713756561279295,
+      "mean_token_accuracy": 0.9926198759675026,
+      "num_tokens": 6121431.0,
+      "step": 1200
+    },
+    {
+      "entropy": 0.02976180042140186,
+      "epoch": 0.9765625,
+      "grad_norm": 0.154296875,
+      "learning_rate": 1.1380208333333333e-05,
+      "loss": 0.02123898983001709,
+      "mean_token_accuracy": 0.992766418159008,
+      "num_tokens": 6379675.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.030388496736995875,
+      "epoch": 1.015625,
+      "grad_norm": 0.1650390625,
+      "learning_rate": 1.0946180555555556e-05,
+      "loss": 0.021283388137817383,
+      "mean_token_accuracy": 0.9927816662192345,
+      "num_tokens": 6635287.0,
+      "step": 1300
+    },
+    {
+      "entropy": 0.029865577281452716,
+      "epoch": 1.0546875,
+      "grad_norm": 0.265625,
+      "learning_rate": 1.0512152777777778e-05,
+      "loss": 0.021030676364898682,
+      "mean_token_accuracy": 0.9929129666090012,
+      "num_tokens": 6888440.0,
+      "step": 1350
+    },
+    {
+      "entropy": 0.031085506100207567,
+      "epoch": 1.09375,
+      "grad_norm": 0.1748046875,
+      "learning_rate": 1.0078125000000001e-05,
+      "loss": 0.02215445041656494,
+      "mean_token_accuracy": 0.9926813915371895,
+      "num_tokens": 7143446.0,
+      "step": 1400
+    },
+    {
+      "entropy": 0.03091464822180569,
+      "epoch": 1.1328125,
+      "grad_norm": 0.2255859375,
+      "learning_rate": 9.644097222222222e-06,
+      "loss": 0.022361652851104738,
+      "mean_token_accuracy": 0.9926716023683548,
+      "num_tokens": 7400487.0,
+      "step": 1450
+    },
+    {
+      "entropy": 0.029652795745059846,
+      "epoch": 1.171875,
+      "grad_norm": 0.130859375,
+      "learning_rate": 9.210069444444446e-06,
+      "loss": 0.02084646940231323,
+      "mean_token_accuracy": 0.9928994616866111,
+      "num_tokens": 7655674.0,
+      "step": 1500
+    },
+    {
+      "entropy": 0.031666285023093224,
+      "epoch": 1.2109375,
+      "grad_norm": 0.1787109375,
+      "learning_rate": 8.776041666666667e-06,
+      "loss": 0.022852597236633302,
+      "mean_token_accuracy": 0.9925497883558273,
+      "num_tokens": 7908433.0,
+      "step": 1550
+    },
+    {
+      "entropy": 0.027730579837225378,
+      "epoch": 1.25,
+      "grad_norm": 0.1689453125,
+      "learning_rate": 8.342013888888889e-06,
+      "loss": 0.019295313358306886,
+      "mean_token_accuracy": 0.9932532203197479,
+      "num_tokens": 8163713.0,
+      "step": 1600
+    },
+    {
+      "entropy": 0.030957318153232338,
+      "epoch": 1.2890625,
+      "grad_norm": 0.1533203125,
+      "learning_rate": 7.907986111111112e-06,
+      "loss": 0.022100534439086914,
+      "mean_token_accuracy": 0.9926850625872612,
+      "num_tokens": 8413661.0,
+      "step": 1650
+    },
+    {
+      "entropy": 0.03025919214822352,
+      "epoch": 1.328125,
+      "grad_norm": 0.19140625,
+      "learning_rate": 7.473958333333334e-06,
+      "loss": 0.021726396083831787,
+      "mean_token_accuracy": 0.9928304460644722,
+      "num_tokens": 8666242.0,
+      "step": 1700
+    },
+    {
+      "entropy": 0.030476445676758886,
+      "epoch": 1.3671875,
+      "grad_norm": 0.17578125,
+      "learning_rate": 7.039930555555556e-06,
+      "loss": 0.0215773606300354,
+      "mean_token_accuracy": 0.9926398959755898,
+      "num_tokens": 8923368.0,
+      "step": 1750
+    },
+    {
+      "entropy": 0.02955903219990432,
+      "epoch": 1.40625,
+      "grad_norm": 0.240234375,
+      "learning_rate": 6.605902777777779e-06,
+      "loss": 0.02073089599609375,
+      "mean_token_accuracy": 0.9930617704987525,
+      "num_tokens": 9177821.0,
+      "step": 1800
+    },
+    {
+      "entropy": 0.029813821725547314,
+      "epoch": 1.4453125,
+      "grad_norm": 0.1640625,
+      "learning_rate": 6.171875e-06,
+      "loss": 0.020715839862823486,
+      "mean_token_accuracy": 0.9928940132260322,
+      "num_tokens": 9430080.0,
+      "step": 1850
+    },
+    {
+      "entropy": 0.027619767771102487,
+      "epoch": 1.484375,
+      "grad_norm": 0.16015625,
+      "learning_rate": 5.737847222222222e-06,
+      "loss": 0.01917331576347351,
+      "mean_token_accuracy": 0.9933555802702904,
+      "num_tokens": 9688541.0,
+      "step": 1900
+    },
+    {
+      "entropy": 0.028464037650264798,
+      "epoch": 1.5234375,
+      "grad_norm": 0.173828125,
+      "learning_rate": 5.303819444444445e-06,
+      "loss": 0.01973416805267334,
+      "mean_token_accuracy": 0.9932170230150222,
+      "num_tokens": 9946814.0,
+      "step": 1950
+    },
+    {
+      "entropy": 0.03081795680336654,
+      "epoch": 1.5625,
+      "grad_norm": 0.296875,
+      "learning_rate": 4.869791666666667e-06,
+      "loss": 0.021953141689300536,
+      "mean_token_accuracy": 0.9926387491822243,
+      "num_tokens": 10201426.0,
+      "step": 2000
+    },
+    {
+      "entropy": 0.030316293751820923,
+      "epoch": 1.6015625,
+      "grad_norm": 0.396484375,
+      "learning_rate": 4.435763888888889e-06,
+      "loss": 0.021486189365386963,
+      "mean_token_accuracy": 0.9927674040198327,
+      "num_tokens": 10454276.0,
+      "step": 2050
+    },
+    {
+      "entropy": 0.02887956439051777,
+      "epoch": 1.640625,
+      "grad_norm": 0.1513671875,
+      "learning_rate": 4.001736111111112e-06,
+      "loss": 0.020056800842285158,
+      "mean_token_accuracy": 0.9931298586726188,
+      "num_tokens": 10707497.0,
+      "step": 2100
+    },
+    {
+      "entropy": 0.028787780185230077,
+      "epoch": 1.6796875,
+      "grad_norm": 0.1455078125,
+      "learning_rate": 3.5677083333333335e-06,
+      "loss": 0.0200783896446228,
+      "mean_token_accuracy": 0.9929500755667686,
+      "num_tokens": 10962955.0,
+      "step": 2150
+    },
+    {
+      "entropy": 0.028631422137841582,
+      "epoch": 1.71875,
+      "grad_norm": 0.1484375,
+      "learning_rate": 3.1336805555555562e-06,
+      "loss": 0.02008913516998291,
+      "mean_token_accuracy": 0.9930686053633689,
+      "num_tokens": 11217333.0,
+      "step": 2200
+    },
+    {
+      "entropy": 0.02850784788839519,
+      "epoch": 1.7578125,
+      "grad_norm": 0.1884765625,
+      "learning_rate": 2.699652777777778e-06,
+      "loss": 0.01996502876281738,
+      "mean_token_accuracy": 0.9930599120259285,
+      "num_tokens": 11472495.0,
+      "step": 2250
+    },
+    {
+      "entropy": 0.028683945639058947,
+      "epoch": 1.796875,
+      "grad_norm": 0.1728515625,
+      "learning_rate": 2.265625e-06,
+      "loss": 0.019921081066131593,
+      "mean_token_accuracy": 0.9932510870695114,
+      "num_tokens": 11728662.0,
+      "step": 2300
+    },
+    {
+      "entropy": 0.02975102465134114,
+      "epoch": 1.8359375,
+      "grad_norm": 0.32421875,
+      "learning_rate": 1.8315972222222223e-06,
+      "loss": 0.020963990688323976,
+      "mean_token_accuracy": 0.9929134699702263,
+      "num_tokens": 11983347.0,
+      "step": 2350
+    },
+    {
+      "entropy": 0.02991143790073693,
+      "epoch": 1.875,
+      "grad_norm": 0.169921875,
+      "learning_rate": 1.3975694444444446e-06,
+      "loss": 0.020808370113372804,
+      "mean_token_accuracy": 0.9929909712076187,
+      "num_tokens": 12236526.0,
+      "step": 2400
+    },
+    {
+      "entropy": 0.030044674500823022,
+      "epoch": 1.9140625,
+      "grad_norm": 0.169921875,
+      "learning_rate": 9.635416666666667e-07,
+      "loss": 0.021207802295684815,
+      "mean_token_accuracy": 0.9927184066176414,
+      "num_tokens": 12489562.0,
+      "step": 2450
+    },
+    {
+      "entropy": 0.02905242417007685,
+      "epoch": 1.953125,
+      "grad_norm": 0.1552734375,
+      "learning_rate": 5.295138888888889e-07,
+      "loss": 0.020578203201293947,
+      "mean_token_accuracy": 0.9929935920238495,
+      "num_tokens": 12748444.0,
+      "step": 2500
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 2560,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.4192422651355955e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26cbf16b2edabc5b8c36be1cccac63caa078ca871b6b4fdcff3ff77d550c19b0
+size 5713

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2116fd642142fda5aa3e5c93e3eed681016140d70428943fa0a8c2b210fa9d1a
+size 6425529112

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:daed434ebef9a6fbc97cbec368443f42c6c1f9a2791d9c2b79844ba66bb27902
+size 5713