commit e9e0e2ab69cf76eb3f6e5648b7aba33e8ef44dba Author: ModelHub XC Date: Sun Jul 26 06:48:09 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: manucif/latamgpt-1b-sft Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..3a4481b --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +--- +license: llama3.2 +base_model: meta-llama/Llama-3.2-1B +language: +- es +- pt +- en +library_name: transformers +pipeline_tag: text-generation +tags: +- sft +- latam +- llama +- latam-gpt +datasets: +- latam-gpt/wikipedia-qa-iberoamerica-clean +- latam-gpt/tulu-3-sft-mixture-no-identity-clean +- latam-gpt/sft-mcqa-unified-clean +- latam-gpt/self-identity-v3-clean +- latam-gpt/copuchatV2-clean +--- + +# latamgpt-1b-sft + +Full supervised fine-tune of `meta-llama/Llama-3.2-1B` on a Latin American instruction mixture, with sequence packing and assistant-only loss masking. + +## Training data + +| Dataset | Role | +|---|---| +| `latam-gpt/wikipedia-qa-iberoamerica-clean` | Iberoamerican QA | +| `latam-gpt/tulu-3-sft-mixture-no-identity-clean` | General SFT mixture | +| `latam-gpt/sft-mcqa-unified-clean` | Multiple-choice QA | +| `latam-gpt/self-identity-v3-clean` | Self-identity | +| `latam-gpt/copuchatV2-clean` | Chilean conversational | + +After tokenization with the Llama 3 chat template and FFD packing at 4096 tokens (99.16% packing efficiency), the training set is **175,995 packed sequences (~721M tokens)** with **119,573 validation examples**. + +## Training setup + +- **Base model:** `meta-llama/Llama-3.2-1B` +- **Method:** Full fine-tune (no LoRA) +- **Sequence length:** 4096, packed with assistant-only loss masking +- **Epochs:** 2 +- **Learning rate:** 1e-5 (cosine, 3% warmup) +- **Batch:** 8 per device × 8 grad accumulation × 2 GPUs = **128 effective** +- **Precision:** bf16 + DeepSpeed ZeRO-2 + gradient checkpointing + Flash Attention 2 +- **Hardware:** 2× H100 80GB +- **Wall time:** ~7.5h + +## Results + +| Metric | Value | +|---|---| +| `train_loss` | 0.8579 | +| `eval_loss` | 0.8910 | +| `epoch` | 2.0 | +| `train_samples` | 175,995 | + +## Usage + +```python +from transformers import AutoTokenizer, AutoModelForCausalLM + +tok = AutoTokenizer.from_pretrained("latam-gpt/latamgpt-1b-sft") +model = AutoModelForCausalLM.from_pretrained("latam-gpt/latamgpt-1b-sft", torch_dtype="bfloat16", device_map="auto") + +messages = [{"role": "user", "content": "¿Cuál es la capital de Chile?"}] +inputs = tok.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt").to(model.device) +out = model.generate(inputs, max_new_tokens=256, do_sample=True, temperature=0.7) +print(tok.decode(out[0][inputs.shape[-1]:], skip_special_tokens=True)) +``` + +## License + +Inherits the [Llama 3.2 Community License](https://www.llama.com/llama3_2/license/) from the base model. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..dee700b --- /dev/null +++ b/all_results.json @@ -0,0 +1,13 @@ +{ + "epoch": 2.0, + "eval_loss": 0.890963077545166, + "eval_runtime": 1952.8654, + "eval_samples_per_second": 61.23, + "eval_steps_per_second": 7.654, + "total_flos": 8.418445674676224e+18, + "train_loss": 0.8579012258703058, + "train_runtime": 27047.1098, + "train_samples": 175995, + "train_samples_per_second": 13.014, + "train_steps_per_second": 0.102 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..78d6037 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,5 @@ +{% for message in messages %}{% if loop.index0 == 0 %}{{ bos_token }}{% endif %}<|start_header_id|>{{ message['role'] }}<|end_header_id|> + +{% if message['role'] == 'assistant' %}{% generation %}{{ message['content'] | trim }}<|eot_id|>{% endgeneration %}{% else %}{{ message['content'] | trim }}<|eot_id|>{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|> + +' }}{% endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..665517a --- /dev/null +++ b/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 128001, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "transformers_version": "4.57.6", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/eval_results.json b/eval_results.json new file mode 100644 index 0000000..cfbaaf1 --- /dev/null +++ b/eval_results.json @@ -0,0 +1,7 @@ +{ + "epoch": 2.0, + "eval_loss": 0.890963077545166, + "eval_runtime": 1952.8654, + "eval_samples_per_second": 61.23, + "eval_steps_per_second": 7.654 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..7b4c4dd --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001 + ], + "pad_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.57.6" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..b3c4e04 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31adb65c631f8a9b1aceae03ef4e63e6c31d1ee636f202dd5c0df3cd5e39d229 +size 2996982344 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..e5b39b6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..3026161 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2063 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..587b311 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 2.0, + "total_flos": 8.418445674676224e+18, + "train_loss": 0.8579012258703058, + "train_runtime": 27047.1098, + "train_samples": 175995, + "train_samples_per_second": 13.014, + "train_steps_per_second": 0.102 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..985abfd --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,1984 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 1000, + "global_step": 2750, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.007272727272727273, + "grad_norm": 24.832788467407227, + "learning_rate": 1.0843373493975905e-06, + "loss": 1.4729, + "step": 10 + }, + { + "epoch": 0.014545454545454545, + "grad_norm": 3.724433422088623, + "learning_rate": 2.2891566265060243e-06, + "loss": 1.314, + "step": 20 + }, + { + "epoch": 0.02181818181818182, + "grad_norm": 1.6110222339630127, + "learning_rate": 3.4939759036144583e-06, + "loss": 1.2551, + "step": 30 + }, + { + "epoch": 0.02909090909090909, + "grad_norm": 0.879056990146637, + "learning_rate": 4.698795180722892e-06, + "loss": 1.1298, + "step": 40 + }, + { + "epoch": 0.03636363636363636, + "grad_norm": 0.8674604892730713, + "learning_rate": 5.9036144578313255e-06, + "loss": 1.0667, + "step": 50 + }, + { + "epoch": 0.04363636363636364, + "grad_norm": 0.8033781051635742, + "learning_rate": 7.1084337349397595e-06, + "loss": 1.0391, + "step": 60 + }, + { + "epoch": 0.05090909090909091, + "grad_norm": 0.8189529180526733, + "learning_rate": 8.313253012048194e-06, + "loss": 1.0478, + "step": 70 + }, + { + "epoch": 0.05818181818181818, + "grad_norm": 0.8405522704124451, + "learning_rate": 9.518072289156628e-06, + "loss": 1.0311, + "step": 80 + }, + { + "epoch": 0.06545454545454546, + "grad_norm": 0.887751579284668, + "learning_rate": 9.977502812148483e-06, + "loss": 1.0091, + "step": 90 + }, + { + "epoch": 0.07272727272727272, + "grad_norm": 0.753106415271759, + "learning_rate": 9.940007499062618e-06, + "loss": 0.9977, + "step": 100 + }, + { + "epoch": 0.08, + "grad_norm": 0.7858726978302002, + "learning_rate": 9.902512185976754e-06, + "loss": 0.9744, + "step": 110 + }, + { + "epoch": 0.08727272727272728, + "grad_norm": 0.6802657842636108, + "learning_rate": 9.86501687289089e-06, + "loss": 0.9672, + "step": 120 + }, + { + "epoch": 0.09454545454545454, + "grad_norm": 0.655599057674408, + "learning_rate": 9.827521559805026e-06, + "loss": 0.961, + "step": 130 + }, + { + "epoch": 0.10181818181818182, + "grad_norm": 0.6806121468544006, + "learning_rate": 9.790026246719161e-06, + "loss": 0.9708, + "step": 140 + }, + { + "epoch": 0.10909090909090909, + "grad_norm": 0.6039880514144897, + "learning_rate": 9.752530933633297e-06, + "loss": 0.9565, + "step": 150 + }, + { + "epoch": 0.11636363636363636, + "grad_norm": 0.6666083931922913, + "learning_rate": 9.715035620547433e-06, + "loss": 0.944, + "step": 160 + }, + { + "epoch": 0.12363636363636364, + "grad_norm": 0.6580015420913696, + "learning_rate": 9.677540307461568e-06, + "loss": 0.9479, + "step": 170 + }, + { + "epoch": 0.13090909090909092, + "grad_norm": 0.6064857244491577, + "learning_rate": 9.640044994375704e-06, + "loss": 0.9489, + "step": 180 + }, + { + "epoch": 0.13818181818181818, + "grad_norm": 0.6560870409011841, + "learning_rate": 9.60254968128984e-06, + "loss": 0.9319, + "step": 190 + }, + { + "epoch": 0.14545454545454545, + "grad_norm": 0.6755111813545227, + "learning_rate": 9.565054368203975e-06, + "loss": 0.9631, + "step": 200 + }, + { + "epoch": 0.15272727272727274, + "grad_norm": 0.6282963156700134, + "learning_rate": 9.52755905511811e-06, + "loss": 0.9085, + "step": 210 + }, + { + "epoch": 0.16, + "grad_norm": 0.6093109250068665, + "learning_rate": 9.490063742032247e-06, + "loss": 0.9323, + "step": 220 + }, + { + "epoch": 0.16727272727272727, + "grad_norm": 0.6879354119300842, + "learning_rate": 9.452568428946382e-06, + "loss": 0.916, + "step": 230 + }, + { + "epoch": 0.17454545454545456, + "grad_norm": 0.6276031136512756, + "learning_rate": 9.415073115860518e-06, + "loss": 0.9021, + "step": 240 + }, + { + "epoch": 0.18181818181818182, + "grad_norm": 0.6175471544265747, + "learning_rate": 9.377577802774654e-06, + "loss": 0.9, + "step": 250 + }, + { + "epoch": 0.1890909090909091, + "grad_norm": 0.6632671356201172, + "learning_rate": 9.34008248968879e-06, + "loss": 0.9258, + "step": 260 + }, + { + "epoch": 0.19636363636363635, + "grad_norm": 0.632043182849884, + "learning_rate": 9.302587176602925e-06, + "loss": 0.9246, + "step": 270 + }, + { + "epoch": 0.20363636363636364, + "grad_norm": 0.6637592315673828, + "learning_rate": 9.265091863517061e-06, + "loss": 0.9287, + "step": 280 + }, + { + "epoch": 0.2109090909090909, + "grad_norm": 0.6319563984870911, + "learning_rate": 9.227596550431196e-06, + "loss": 0.9322, + "step": 290 + }, + { + "epoch": 0.21818181818181817, + "grad_norm": 0.6026636362075806, + "learning_rate": 9.190101237345333e-06, + "loss": 0.911, + "step": 300 + }, + { + "epoch": 0.22545454545454546, + "grad_norm": 0.6266274452209473, + "learning_rate": 9.152605924259468e-06, + "loss": 0.9288, + "step": 310 + }, + { + "epoch": 0.23272727272727273, + "grad_norm": 0.6260218024253845, + "learning_rate": 9.115110611173603e-06, + "loss": 0.89, + "step": 320 + }, + { + "epoch": 0.24, + "grad_norm": 0.619553804397583, + "learning_rate": 9.07761529808774e-06, + "loss": 0.9066, + "step": 330 + }, + { + "epoch": 0.24727272727272728, + "grad_norm": 0.6455956101417542, + "learning_rate": 9.040119985001875e-06, + "loss": 0.8969, + "step": 340 + }, + { + "epoch": 0.2545454545454545, + "grad_norm": 0.6297126412391663, + "learning_rate": 9.002624671916012e-06, + "loss": 0.9087, + "step": 350 + }, + { + "epoch": 0.26181818181818184, + "grad_norm": 0.6240535378456116, + "learning_rate": 8.965129358830147e-06, + "loss": 0.903, + "step": 360 + }, + { + "epoch": 0.2690909090909091, + "grad_norm": 0.6323297023773193, + "learning_rate": 8.927634045744282e-06, + "loss": 0.9021, + "step": 370 + }, + { + "epoch": 0.27636363636363637, + "grad_norm": 0.629094660282135, + "learning_rate": 8.89013873265842e-06, + "loss": 0.8945, + "step": 380 + }, + { + "epoch": 0.28363636363636363, + "grad_norm": 0.6123955249786377, + "learning_rate": 8.852643419572554e-06, + "loss": 0.8997, + "step": 390 + }, + { + "epoch": 0.2909090909090909, + "grad_norm": 0.6139543056488037, + "learning_rate": 8.81514810648669e-06, + "loss": 0.907, + "step": 400 + }, + { + "epoch": 0.29818181818181816, + "grad_norm": 0.6207508444786072, + "learning_rate": 8.777652793400826e-06, + "loss": 0.9011, + "step": 410 + }, + { + "epoch": 0.3054545454545455, + "grad_norm": 0.6085091233253479, + "learning_rate": 8.740157480314961e-06, + "loss": 0.895, + "step": 420 + }, + { + "epoch": 0.31272727272727274, + "grad_norm": 0.6231858134269714, + "learning_rate": 8.702662167229096e-06, + "loss": 0.9153, + "step": 430 + }, + { + "epoch": 0.32, + "grad_norm": 0.6155027747154236, + "learning_rate": 8.665166854143233e-06, + "loss": 0.8809, + "step": 440 + }, + { + "epoch": 0.32727272727272727, + "grad_norm": 0.6154032349586487, + "learning_rate": 8.627671541057368e-06, + "loss": 0.879, + "step": 450 + }, + { + "epoch": 0.33454545454545453, + "grad_norm": 0.6161757111549377, + "learning_rate": 8.590176227971503e-06, + "loss": 0.9212, + "step": 460 + }, + { + "epoch": 0.3418181818181818, + "grad_norm": 0.6051761507987976, + "learning_rate": 8.55268091488564e-06, + "loss": 0.9021, + "step": 470 + }, + { + "epoch": 0.3490909090909091, + "grad_norm": 0.6489289999008179, + "learning_rate": 8.515185601799775e-06, + "loss": 0.8902, + "step": 480 + }, + { + "epoch": 0.3563636363636364, + "grad_norm": 0.6450833082199097, + "learning_rate": 8.47769028871391e-06, + "loss": 0.9078, + "step": 490 + }, + { + "epoch": 0.36363636363636365, + "grad_norm": 0.5917351841926575, + "learning_rate": 8.440194975628047e-06, + "loss": 0.884, + "step": 500 + }, + { + "epoch": 0.3709090909090909, + "grad_norm": 0.6286096572875977, + "learning_rate": 8.402699662542182e-06, + "loss": 0.8955, + "step": 510 + }, + { + "epoch": 0.3781818181818182, + "grad_norm": 0.6284880042076111, + "learning_rate": 8.365204349456319e-06, + "loss": 0.8909, + "step": 520 + }, + { + "epoch": 0.38545454545454544, + "grad_norm": 0.6258820295333862, + "learning_rate": 8.327709036370454e-06, + "loss": 0.8852, + "step": 530 + }, + { + "epoch": 0.3927272727272727, + "grad_norm": 0.6000548005104065, + "learning_rate": 8.290213723284591e-06, + "loss": 0.865, + "step": 540 + }, + { + "epoch": 0.4, + "grad_norm": 0.6457489728927612, + "learning_rate": 8.252718410198726e-06, + "loss": 0.8927, + "step": 550 + }, + { + "epoch": 0.4072727272727273, + "grad_norm": 0.6275245547294617, + "learning_rate": 8.215223097112861e-06, + "loss": 0.88, + "step": 560 + }, + { + "epoch": 0.41454545454545455, + "grad_norm": 0.6320298910140991, + "learning_rate": 8.177727784026998e-06, + "loss": 0.8886, + "step": 570 + }, + { + "epoch": 0.4218181818181818, + "grad_norm": 0.6126200556755066, + "learning_rate": 8.140232470941133e-06, + "loss": 0.8715, + "step": 580 + }, + { + "epoch": 0.4290909090909091, + "grad_norm": 0.5955527424812317, + "learning_rate": 8.102737157855268e-06, + "loss": 0.8746, + "step": 590 + }, + { + "epoch": 0.43636363636363634, + "grad_norm": 0.6173805594444275, + "learning_rate": 8.065241844769405e-06, + "loss": 0.8798, + "step": 600 + }, + { + "epoch": 0.44363636363636366, + "grad_norm": 0.6467263698577881, + "learning_rate": 8.02774653168354e-06, + "loss": 0.8815, + "step": 610 + }, + { + "epoch": 0.4509090909090909, + "grad_norm": 0.6411978602409363, + "learning_rate": 7.990251218597675e-06, + "loss": 0.8823, + "step": 620 + }, + { + "epoch": 0.4581818181818182, + "grad_norm": 0.6345722079277039, + "learning_rate": 7.952755905511812e-06, + "loss": 0.8874, + "step": 630 + }, + { + "epoch": 0.46545454545454545, + "grad_norm": 0.6099885702133179, + "learning_rate": 7.915260592425947e-06, + "loss": 0.8809, + "step": 640 + }, + { + "epoch": 0.4727272727272727, + "grad_norm": 0.623310923576355, + "learning_rate": 7.877765279340082e-06, + "loss": 0.8843, + "step": 650 + }, + { + "epoch": 0.48, + "grad_norm": 0.6418883204460144, + "learning_rate": 7.840269966254219e-06, + "loss": 0.8817, + "step": 660 + }, + { + "epoch": 0.48727272727272725, + "grad_norm": 0.5989499688148499, + "learning_rate": 7.802774653168354e-06, + "loss": 0.8867, + "step": 670 + }, + { + "epoch": 0.49454545454545457, + "grad_norm": 0.6249462366104126, + "learning_rate": 7.76527934008249e-06, + "loss": 0.8913, + "step": 680 + }, + { + "epoch": 0.5018181818181818, + "grad_norm": 0.6270976662635803, + "learning_rate": 7.727784026996626e-06, + "loss": 0.8735, + "step": 690 + }, + { + "epoch": 0.509090909090909, + "grad_norm": 0.6204495429992676, + "learning_rate": 7.690288713910761e-06, + "loss": 0.8723, + "step": 700 + }, + { + "epoch": 0.5163636363636364, + "grad_norm": 0.6200954914093018, + "learning_rate": 7.652793400824898e-06, + "loss": 0.8791, + "step": 710 + }, + { + "epoch": 0.5236363636363637, + "grad_norm": 0.6141085624694824, + "learning_rate": 7.615298087739034e-06, + "loss": 0.8731, + "step": 720 + }, + { + "epoch": 0.5309090909090909, + "grad_norm": 0.657077431678772, + "learning_rate": 7.577802774653169e-06, + "loss": 0.8877, + "step": 730 + }, + { + "epoch": 0.5381818181818182, + "grad_norm": 0.6287103891372681, + "learning_rate": 7.540307461567304e-06, + "loss": 0.8688, + "step": 740 + }, + { + "epoch": 0.5454545454545454, + "grad_norm": 0.6096076369285583, + "learning_rate": 7.502812148481441e-06, + "loss": 0.8837, + "step": 750 + }, + { + "epoch": 0.5527272727272727, + "grad_norm": 0.6509519815444946, + "learning_rate": 7.465316835395576e-06, + "loss": 0.8833, + "step": 760 + }, + { + "epoch": 0.56, + "grad_norm": 0.6012381315231323, + "learning_rate": 7.427821522309712e-06, + "loss": 0.8587, + "step": 770 + }, + { + "epoch": 0.5672727272727273, + "grad_norm": 0.6379227638244629, + "learning_rate": 7.390326209223848e-06, + "loss": 0.8659, + "step": 780 + }, + { + "epoch": 0.5745454545454546, + "grad_norm": 0.5909913778305054, + "learning_rate": 7.352830896137984e-06, + "loss": 0.8822, + "step": 790 + }, + { + "epoch": 0.5818181818181818, + "grad_norm": 0.600817084312439, + "learning_rate": 7.315335583052119e-06, + "loss": 0.8781, + "step": 800 + }, + { + "epoch": 0.5890909090909091, + "grad_norm": 0.603955090045929, + "learning_rate": 7.277840269966256e-06, + "loss": 0.8715, + "step": 810 + }, + { + "epoch": 0.5963636363636363, + "grad_norm": 0.5938793420791626, + "learning_rate": 7.240344956880391e-06, + "loss": 0.8792, + "step": 820 + }, + { + "epoch": 0.6036363636363636, + "grad_norm": 0.6032233834266663, + "learning_rate": 7.202849643794526e-06, + "loss": 0.8911, + "step": 830 + }, + { + "epoch": 0.610909090909091, + "grad_norm": 0.6159318089485168, + "learning_rate": 7.165354330708662e-06, + "loss": 0.8579, + "step": 840 + }, + { + "epoch": 0.6181818181818182, + "grad_norm": 0.6361089944839478, + "learning_rate": 7.127859017622798e-06, + "loss": 0.8528, + "step": 850 + }, + { + "epoch": 0.6254545454545455, + "grad_norm": 0.6163432598114014, + "learning_rate": 7.090363704536934e-06, + "loss": 0.8685, + "step": 860 + }, + { + "epoch": 0.6327272727272727, + "grad_norm": 0.5784986615180969, + "learning_rate": 7.052868391451069e-06, + "loss": 0.8654, + "step": 870 + }, + { + "epoch": 0.64, + "grad_norm": 0.6399606466293335, + "learning_rate": 7.015373078365206e-06, + "loss": 0.8681, + "step": 880 + }, + { + "epoch": 0.6472727272727272, + "grad_norm": 0.6454589366912842, + "learning_rate": 6.977877765279341e-06, + "loss": 0.8755, + "step": 890 + }, + { + "epoch": 0.6545454545454545, + "grad_norm": 0.6353079080581665, + "learning_rate": 6.940382452193476e-06, + "loss": 0.8773, + "step": 900 + }, + { + "epoch": 0.6618181818181819, + "grad_norm": 0.5652336478233337, + "learning_rate": 6.902887139107613e-06, + "loss": 0.8494, + "step": 910 + }, + { + "epoch": 0.6690909090909091, + "grad_norm": 0.5925925374031067, + "learning_rate": 6.865391826021748e-06, + "loss": 0.8566, + "step": 920 + }, + { + "epoch": 0.6763636363636364, + "grad_norm": 0.6264874339103699, + "learning_rate": 6.827896512935883e-06, + "loss": 0.8755, + "step": 930 + }, + { + "epoch": 0.6836363636363636, + "grad_norm": 0.6029939651489258, + "learning_rate": 6.79040119985002e-06, + "loss": 0.8815, + "step": 940 + }, + { + "epoch": 0.6909090909090909, + "grad_norm": 0.6007083654403687, + "learning_rate": 6.752905886764155e-06, + "loss": 0.8688, + "step": 950 + }, + { + "epoch": 0.6981818181818182, + "grad_norm": 0.5800495147705078, + "learning_rate": 6.715410573678291e-06, + "loss": 0.8732, + "step": 960 + }, + { + "epoch": 0.7054545454545454, + "grad_norm": 0.5926985740661621, + "learning_rate": 6.677915260592427e-06, + "loss": 0.8719, + "step": 970 + }, + { + "epoch": 0.7127272727272728, + "grad_norm": 0.6431898474693298, + "learning_rate": 6.6404199475065626e-06, + "loss": 0.8808, + "step": 980 + }, + { + "epoch": 0.72, + "grad_norm": 0.5893067717552185, + "learning_rate": 6.602924634420698e-06, + "loss": 0.8571, + "step": 990 + }, + { + "epoch": 0.7272727272727273, + "grad_norm": 0.592795729637146, + "learning_rate": 6.5654293213348345e-06, + "loss": 0.8672, + "step": 1000 + }, + { + "epoch": 0.7272727272727273, + "eval_loss": 0.919316291809082, + "eval_runtime": 2141.287, + "eval_samples_per_second": 55.842, + "eval_steps_per_second": 6.98, + "step": 1000 + }, + { + "epoch": 0.7345454545454545, + "grad_norm": 0.6309456825256348, + "learning_rate": 6.5279340082489696e-06, + "loss": 0.8528, + "step": 1010 + }, + { + "epoch": 0.7418181818181818, + "grad_norm": 0.6349218487739563, + "learning_rate": 6.490438695163105e-06, + "loss": 0.8513, + "step": 1020 + }, + { + "epoch": 0.7490909090909091, + "grad_norm": 0.5786005258560181, + "learning_rate": 6.4529433820772415e-06, + "loss": 0.8856, + "step": 1030 + }, + { + "epoch": 0.7563636363636363, + "grad_norm": 0.5970295667648315, + "learning_rate": 6.415448068991377e-06, + "loss": 0.8567, + "step": 1040 + }, + { + "epoch": 0.7636363636363637, + "grad_norm": 0.5964470505714417, + "learning_rate": 6.3779527559055125e-06, + "loss": 0.8656, + "step": 1050 + }, + { + "epoch": 0.7709090909090909, + "grad_norm": 0.6192710995674133, + "learning_rate": 6.3404574428196485e-06, + "loss": 0.8496, + "step": 1060 + }, + { + "epoch": 0.7781818181818182, + "grad_norm": 0.5974547863006592, + "learning_rate": 6.302962129733784e-06, + "loss": 0.8534, + "step": 1070 + }, + { + "epoch": 0.7854545454545454, + "grad_norm": 0.5947730541229248, + "learning_rate": 6.2654668166479195e-06, + "loss": 0.8524, + "step": 1080 + }, + { + "epoch": 0.7927272727272727, + "grad_norm": 0.6238484978675842, + "learning_rate": 6.227971503562055e-06, + "loss": 0.8605, + "step": 1090 + }, + { + "epoch": 0.8, + "grad_norm": 0.6511428952217102, + "learning_rate": 6.1904761904761914e-06, + "loss": 0.867, + "step": 1100 + }, + { + "epoch": 0.8072727272727273, + "grad_norm": 0.6051254868507385, + "learning_rate": 6.1529808773903265e-06, + "loss": 0.8759, + "step": 1110 + }, + { + "epoch": 0.8145454545454546, + "grad_norm": 0.6452696323394775, + "learning_rate": 6.115485564304462e-06, + "loss": 0.8839, + "step": 1120 + }, + { + "epoch": 0.8218181818181818, + "grad_norm": 0.6039929389953613, + "learning_rate": 6.0779902512185984e-06, + "loss": 0.8465, + "step": 1130 + }, + { + "epoch": 0.8290909090909091, + "grad_norm": 0.6003919243812561, + "learning_rate": 6.0404949381327335e-06, + "loss": 0.8308, + "step": 1140 + }, + { + "epoch": 0.8363636363636363, + "grad_norm": 0.6162687540054321, + "learning_rate": 6.0029996250468695e-06, + "loss": 0.8835, + "step": 1150 + }, + { + "epoch": 0.8436363636363636, + "grad_norm": 0.6076496243476868, + "learning_rate": 5.9655043119610054e-06, + "loss": 0.8677, + "step": 1160 + }, + { + "epoch": 0.850909090909091, + "grad_norm": 0.6274608969688416, + "learning_rate": 5.928008998875141e-06, + "loss": 0.8705, + "step": 1170 + }, + { + "epoch": 0.8581818181818182, + "grad_norm": 0.6353155970573425, + "learning_rate": 5.8905136857892765e-06, + "loss": 0.8519, + "step": 1180 + }, + { + "epoch": 0.8654545454545455, + "grad_norm": 0.6303426623344421, + "learning_rate": 5.853018372703413e-06, + "loss": 0.8536, + "step": 1190 + }, + { + "epoch": 0.8727272727272727, + "grad_norm": 0.607300877571106, + "learning_rate": 5.815523059617548e-06, + "loss": 0.8695, + "step": 1200 + }, + { + "epoch": 0.88, + "grad_norm": 0.6049074530601501, + "learning_rate": 5.7780277465316835e-06, + "loss": 0.8722, + "step": 1210 + }, + { + "epoch": 0.8872727272727273, + "grad_norm": 0.5959789156913757, + "learning_rate": 5.74053243344582e-06, + "loss": 0.8507, + "step": 1220 + }, + { + "epoch": 0.8945454545454545, + "grad_norm": 0.5832094550132751, + "learning_rate": 5.703037120359955e-06, + "loss": 0.858, + "step": 1230 + }, + { + "epoch": 0.9018181818181819, + "grad_norm": 0.5952485799789429, + "learning_rate": 5.6655418072740905e-06, + "loss": 0.8521, + "step": 1240 + }, + { + "epoch": 0.9090909090909091, + "grad_norm": 0.5914123058319092, + "learning_rate": 5.628046494188227e-06, + "loss": 0.8564, + "step": 1250 + }, + { + "epoch": 0.9163636363636364, + "grad_norm": 0.5735720992088318, + "learning_rate": 5.590551181102362e-06, + "loss": 0.8721, + "step": 1260 + }, + { + "epoch": 0.9236363636363636, + "grad_norm": 0.6073433756828308, + "learning_rate": 5.553055868016498e-06, + "loss": 0.8653, + "step": 1270 + }, + { + "epoch": 0.9309090909090909, + "grad_norm": 0.5949496626853943, + "learning_rate": 5.515560554930634e-06, + "loss": 0.8665, + "step": 1280 + }, + { + "epoch": 0.9381818181818182, + "grad_norm": 0.6578984260559082, + "learning_rate": 5.47806524184477e-06, + "loss": 0.8544, + "step": 1290 + }, + { + "epoch": 0.9454545454545454, + "grad_norm": 0.6195460557937622, + "learning_rate": 5.440569928758905e-06, + "loss": 0.8561, + "step": 1300 + }, + { + "epoch": 0.9527272727272728, + "grad_norm": 0.5933972597122192, + "learning_rate": 5.403074615673042e-06, + "loss": 0.863, + "step": 1310 + }, + { + "epoch": 0.96, + "grad_norm": 0.5625194311141968, + "learning_rate": 5.365579302587177e-06, + "loss": 0.8698, + "step": 1320 + }, + { + "epoch": 0.9672727272727273, + "grad_norm": 0.5966600775718689, + "learning_rate": 5.328083989501312e-06, + "loss": 0.8412, + "step": 1330 + }, + { + "epoch": 0.9745454545454545, + "grad_norm": 0.6089814901351929, + "learning_rate": 5.290588676415449e-06, + "loss": 0.853, + "step": 1340 + }, + { + "epoch": 0.9818181818181818, + "grad_norm": 0.5774534344673157, + "learning_rate": 5.253093363329584e-06, + "loss": 0.8682, + "step": 1350 + }, + { + "epoch": 0.9890909090909091, + "grad_norm": 0.6104735732078552, + "learning_rate": 5.21559805024372e-06, + "loss": 0.8553, + "step": 1360 + }, + { + "epoch": 0.9963636363636363, + "grad_norm": 0.6288988590240479, + "learning_rate": 5.178102737157855e-06, + "loss": 0.8257, + "step": 1370 + }, + { + "epoch": 1.0036363636363637, + "grad_norm": 0.6185275316238403, + "learning_rate": 5.140607424071991e-06, + "loss": 0.8182, + "step": 1380 + }, + { + "epoch": 1.010909090909091, + "grad_norm": 0.6155534386634827, + "learning_rate": 5.103112110986127e-06, + "loss": 0.8235, + "step": 1390 + }, + { + "epoch": 1.018181818181818, + "grad_norm": 0.6087826490402222, + "learning_rate": 5.065616797900262e-06, + "loss": 0.8182, + "step": 1400 + }, + { + "epoch": 1.0254545454545454, + "grad_norm": 0.5803642272949219, + "learning_rate": 5.028121484814399e-06, + "loss": 0.8218, + "step": 1410 + }, + { + "epoch": 1.0327272727272727, + "grad_norm": 0.6309434771537781, + "learning_rate": 4.990626171728534e-06, + "loss": 0.8125, + "step": 1420 + }, + { + "epoch": 1.04, + "grad_norm": 0.6362752914428711, + "learning_rate": 4.95313085864267e-06, + "loss": 0.8066, + "step": 1430 + }, + { + "epoch": 1.0472727272727274, + "grad_norm": 0.5848464369773865, + "learning_rate": 4.915635545556805e-06, + "loss": 0.8156, + "step": 1440 + }, + { + "epoch": 1.0545454545454545, + "grad_norm": 0.619362473487854, + "learning_rate": 4.878140232470941e-06, + "loss": 0.8158, + "step": 1450 + }, + { + "epoch": 1.0618181818181818, + "grad_norm": 0.5853692889213562, + "learning_rate": 4.840644919385077e-06, + "loss": 0.8319, + "step": 1460 + }, + { + "epoch": 1.069090909090909, + "grad_norm": 0.607729434967041, + "learning_rate": 4.803149606299213e-06, + "loss": 0.8145, + "step": 1470 + }, + { + "epoch": 1.0763636363636364, + "grad_norm": 0.5888605117797852, + "learning_rate": 4.765654293213349e-06, + "loss": 0.8114, + "step": 1480 + }, + { + "epoch": 1.0836363636363637, + "grad_norm": 0.6180461049079895, + "learning_rate": 4.728158980127485e-06, + "loss": 0.8234, + "step": 1490 + }, + { + "epoch": 1.0909090909090908, + "grad_norm": 0.5963523387908936, + "learning_rate": 4.69066366704162e-06, + "loss": 0.8159, + "step": 1500 + }, + { + "epoch": 1.0981818181818181, + "grad_norm": 0.6214097738265991, + "learning_rate": 4.653168353955756e-06, + "loss": 0.8256, + "step": 1510 + }, + { + "epoch": 1.1054545454545455, + "grad_norm": 0.5866185426712036, + "learning_rate": 4.615673040869892e-06, + "loss": 0.8431, + "step": 1520 + }, + { + "epoch": 1.1127272727272728, + "grad_norm": 0.5828819870948792, + "learning_rate": 4.578177727784027e-06, + "loss": 0.8064, + "step": 1530 + }, + { + "epoch": 1.12, + "grad_norm": 0.5965102910995483, + "learning_rate": 4.540682414698163e-06, + "loss": 0.8281, + "step": 1540 + }, + { + "epoch": 1.1272727272727272, + "grad_norm": 0.5992522239685059, + "learning_rate": 4.503187101612299e-06, + "loss": 0.8134, + "step": 1550 + }, + { + "epoch": 1.1345454545454545, + "grad_norm": 0.6310180425643921, + "learning_rate": 4.465691788526434e-06, + "loss": 0.819, + "step": 1560 + }, + { + "epoch": 1.1418181818181818, + "grad_norm": 0.588926374912262, + "learning_rate": 4.42819647544057e-06, + "loss": 0.8333, + "step": 1570 + }, + { + "epoch": 1.1490909090909092, + "grad_norm": 0.5981424450874329, + "learning_rate": 4.390701162354706e-06, + "loss": 0.8386, + "step": 1580 + }, + { + "epoch": 1.1563636363636363, + "grad_norm": 0.634250283241272, + "learning_rate": 4.353205849268842e-06, + "loss": 0.8034, + "step": 1590 + }, + { + "epoch": 1.1636363636363636, + "grad_norm": 0.5827885866165161, + "learning_rate": 4.315710536182978e-06, + "loss": 0.8288, + "step": 1600 + }, + { + "epoch": 1.170909090909091, + "grad_norm": 0.6020681858062744, + "learning_rate": 4.278215223097113e-06, + "loss": 0.8181, + "step": 1610 + }, + { + "epoch": 1.1781818181818182, + "grad_norm": 0.5973985195159912, + "learning_rate": 4.240719910011249e-06, + "loss": 0.8206, + "step": 1620 + }, + { + "epoch": 1.1854545454545455, + "grad_norm": 0.5879353880882263, + "learning_rate": 4.203224596925385e-06, + "loss": 0.8129, + "step": 1630 + }, + { + "epoch": 1.1927272727272726, + "grad_norm": 0.6243763566017151, + "learning_rate": 4.16572928383952e-06, + "loss": 0.8067, + "step": 1640 + }, + { + "epoch": 1.2, + "grad_norm": 0.583442211151123, + "learning_rate": 4.128233970753656e-06, + "loss": 0.8279, + "step": 1650 + }, + { + "epoch": 1.2072727272727273, + "grad_norm": 0.6032597422599792, + "learning_rate": 4.090738657667792e-06, + "loss": 0.8115, + "step": 1660 + }, + { + "epoch": 1.2145454545454546, + "grad_norm": 0.6115988492965698, + "learning_rate": 4.053243344581928e-06, + "loss": 0.8221, + "step": 1670 + }, + { + "epoch": 1.221818181818182, + "grad_norm": 0.600167453289032, + "learning_rate": 4.015748031496064e-06, + "loss": 0.8008, + "step": 1680 + }, + { + "epoch": 1.229090909090909, + "grad_norm": 0.6090269088745117, + "learning_rate": 3.978252718410199e-06, + "loss": 0.8196, + "step": 1690 + }, + { + "epoch": 1.2363636363636363, + "grad_norm": 0.598509669303894, + "learning_rate": 3.940757405324335e-06, + "loss": 0.8168, + "step": 1700 + }, + { + "epoch": 1.2436363636363637, + "grad_norm": 0.5862186551094055, + "learning_rate": 3.903262092238471e-06, + "loss": 0.819, + "step": 1710 + }, + { + "epoch": 1.250909090909091, + "grad_norm": 0.6207866072654724, + "learning_rate": 3.865766779152606e-06, + "loss": 0.8074, + "step": 1720 + }, + { + "epoch": 1.2581818181818183, + "grad_norm": 0.5965321063995361, + "learning_rate": 3.828271466066742e-06, + "loss": 0.8149, + "step": 1730 + }, + { + "epoch": 1.2654545454545454, + "grad_norm": 0.6035894751548767, + "learning_rate": 3.790776152980878e-06, + "loss": 0.8347, + "step": 1740 + }, + { + "epoch": 1.2727272727272727, + "grad_norm": 0.5681039094924927, + "learning_rate": 3.7532808398950133e-06, + "loss": 0.8038, + "step": 1750 + }, + { + "epoch": 1.28, + "grad_norm": 0.6101064085960388, + "learning_rate": 3.7157855268091493e-06, + "loss": 0.7951, + "step": 1760 + }, + { + "epoch": 1.2872727272727273, + "grad_norm": 0.5739514827728271, + "learning_rate": 3.6782902137232852e-06, + "loss": 0.8189, + "step": 1770 + }, + { + "epoch": 1.2945454545454544, + "grad_norm": 0.5720605254173279, + "learning_rate": 3.6407949006374203e-06, + "loss": 0.8021, + "step": 1780 + }, + { + "epoch": 1.3018181818181818, + "grad_norm": 0.5951290726661682, + "learning_rate": 3.6032995875515563e-06, + "loss": 0.7991, + "step": 1790 + }, + { + "epoch": 1.309090909090909, + "grad_norm": 0.5760191679000854, + "learning_rate": 3.5658042744656922e-06, + "loss": 0.8131, + "step": 1800 + }, + { + "epoch": 1.3163636363636364, + "grad_norm": 0.6369950771331787, + "learning_rate": 3.5283089613798277e-06, + "loss": 0.8319, + "step": 1810 + }, + { + "epoch": 1.3236363636363637, + "grad_norm": 0.6073457598686218, + "learning_rate": 3.4908136482939637e-06, + "loss": 0.8173, + "step": 1820 + }, + { + "epoch": 1.330909090909091, + "grad_norm": 0.6086942553520203, + "learning_rate": 3.453318335208099e-06, + "loss": 0.8218, + "step": 1830 + }, + { + "epoch": 1.3381818181818181, + "grad_norm": 0.587218165397644, + "learning_rate": 3.4158230221222347e-06, + "loss": 0.8167, + "step": 1840 + }, + { + "epoch": 1.3454545454545455, + "grad_norm": 0.5894433856010437, + "learning_rate": 3.3783277090363707e-06, + "loss": 0.8253, + "step": 1850 + }, + { + "epoch": 1.3527272727272728, + "grad_norm": 0.5601096749305725, + "learning_rate": 3.3408323959505062e-06, + "loss": 0.801, + "step": 1860 + }, + { + "epoch": 1.3599999999999999, + "grad_norm": 0.6079325079917908, + "learning_rate": 3.303337082864642e-06, + "loss": 0.8117, + "step": 1870 + }, + { + "epoch": 1.3672727272727272, + "grad_norm": 0.581007182598114, + "learning_rate": 3.265841769778778e-06, + "loss": 0.8241, + "step": 1880 + }, + { + "epoch": 1.3745454545454545, + "grad_norm": 0.5827407836914062, + "learning_rate": 3.2283464566929136e-06, + "loss": 0.8193, + "step": 1890 + }, + { + "epoch": 1.3818181818181818, + "grad_norm": 0.5792385935783386, + "learning_rate": 3.1908511436070496e-06, + "loss": 0.8107, + "step": 1900 + }, + { + "epoch": 1.3890909090909092, + "grad_norm": 0.5755916833877563, + "learning_rate": 3.153355830521185e-06, + "loss": 0.8083, + "step": 1910 + }, + { + "epoch": 1.3963636363636365, + "grad_norm": 0.6043480634689331, + "learning_rate": 3.1158605174353206e-06, + "loss": 0.8047, + "step": 1920 + }, + { + "epoch": 1.4036363636363636, + "grad_norm": 0.5923509001731873, + "learning_rate": 3.0783652043494566e-06, + "loss": 0.8264, + "step": 1930 + }, + { + "epoch": 1.410909090909091, + "grad_norm": 0.5509864091873169, + "learning_rate": 3.0408698912635925e-06, + "loss": 0.8219, + "step": 1940 + }, + { + "epoch": 1.4181818181818182, + "grad_norm": 0.6133068799972534, + "learning_rate": 3.003374578177728e-06, + "loss": 0.8303, + "step": 1950 + }, + { + "epoch": 1.4254545454545455, + "grad_norm": 0.5911954045295715, + "learning_rate": 2.965879265091864e-06, + "loss": 0.8245, + "step": 1960 + }, + { + "epoch": 1.4327272727272726, + "grad_norm": 0.609562337398529, + "learning_rate": 2.928383952005999e-06, + "loss": 0.7991, + "step": 1970 + }, + { + "epoch": 1.44, + "grad_norm": 0.5898092985153198, + "learning_rate": 2.890888638920135e-06, + "loss": 0.8264, + "step": 1980 + }, + { + "epoch": 1.4472727272727273, + "grad_norm": 0.6039696931838989, + "learning_rate": 2.853393325834271e-06, + "loss": 0.8176, + "step": 1990 + }, + { + "epoch": 1.4545454545454546, + "grad_norm": 0.6092890501022339, + "learning_rate": 2.8158980127484065e-06, + "loss": 0.8101, + "step": 2000 + }, + { + "epoch": 1.4545454545454546, + "eval_loss": 0.8971814513206482, + "eval_runtime": 2118.2926, + "eval_samples_per_second": 56.448, + "eval_steps_per_second": 7.056, + "step": 2000 + }, + { + "epoch": 1.461818181818182, + "grad_norm": 0.6092020273208618, + "learning_rate": 2.7784026996625425e-06, + "loss": 0.8359, + "step": 2010 + }, + { + "epoch": 1.4690909090909092, + "grad_norm": 0.6030200719833374, + "learning_rate": 2.7409073865766784e-06, + "loss": 0.7967, + "step": 2020 + }, + { + "epoch": 1.4763636363636363, + "grad_norm": 0.5874107480049133, + "learning_rate": 2.7034120734908135e-06, + "loss": 0.8141, + "step": 2030 + }, + { + "epoch": 1.4836363636363636, + "grad_norm": 0.6113495826721191, + "learning_rate": 2.6659167604049495e-06, + "loss": 0.8291, + "step": 2040 + }, + { + "epoch": 1.490909090909091, + "grad_norm": 0.5739171504974365, + "learning_rate": 2.6284214473190854e-06, + "loss": 0.8159, + "step": 2050 + }, + { + "epoch": 1.498181818181818, + "grad_norm": 0.5858970880508423, + "learning_rate": 2.590926134233221e-06, + "loss": 0.822, + "step": 2060 + }, + { + "epoch": 1.5054545454545454, + "grad_norm": 0.5980799794197083, + "learning_rate": 2.553430821147357e-06, + "loss": 0.8052, + "step": 2070 + }, + { + "epoch": 1.5127272727272727, + "grad_norm": 0.5883092284202576, + "learning_rate": 2.515935508061493e-06, + "loss": 0.8125, + "step": 2080 + }, + { + "epoch": 1.52, + "grad_norm": 0.5935123562812805, + "learning_rate": 2.478440194975628e-06, + "loss": 0.7978, + "step": 2090 + }, + { + "epoch": 1.5272727272727273, + "grad_norm": 0.5738603472709656, + "learning_rate": 2.440944881889764e-06, + "loss": 0.8352, + "step": 2100 + }, + { + "epoch": 1.5345454545454547, + "grad_norm": 0.5987142324447632, + "learning_rate": 2.4034495688039e-06, + "loss": 0.8151, + "step": 2110 + }, + { + "epoch": 1.541818181818182, + "grad_norm": 0.5858705639839172, + "learning_rate": 2.3659542557180354e-06, + "loss": 0.8258, + "step": 2120 + }, + { + "epoch": 1.549090909090909, + "grad_norm": 0.5583072900772095, + "learning_rate": 2.328458942632171e-06, + "loss": 0.8126, + "step": 2130 + }, + { + "epoch": 1.5563636363636364, + "grad_norm": 0.5778517723083496, + "learning_rate": 2.290963629546307e-06, + "loss": 0.7926, + "step": 2140 + }, + { + "epoch": 1.5636363636363635, + "grad_norm": 0.5871937274932861, + "learning_rate": 2.253468316460443e-06, + "loss": 0.8216, + "step": 2150 + }, + { + "epoch": 1.5709090909090908, + "grad_norm": 0.581562876701355, + "learning_rate": 2.2159730033745783e-06, + "loss": 0.8096, + "step": 2160 + }, + { + "epoch": 1.5781818181818181, + "grad_norm": 0.5970920324325562, + "learning_rate": 2.1784776902887143e-06, + "loss": 0.8306, + "step": 2170 + }, + { + "epoch": 1.5854545454545454, + "grad_norm": 0.5595991015434265, + "learning_rate": 2.14098237720285e-06, + "loss": 0.8053, + "step": 2180 + }, + { + "epoch": 1.5927272727272728, + "grad_norm": 0.592958927154541, + "learning_rate": 2.1034870641169853e-06, + "loss": 0.7871, + "step": 2190 + }, + { + "epoch": 1.6, + "grad_norm": 0.5841421484947205, + "learning_rate": 2.0659917510311213e-06, + "loss": 0.8051, + "step": 2200 + }, + { + "epoch": 1.6072727272727274, + "grad_norm": 0.577006995677948, + "learning_rate": 2.0284964379452572e-06, + "loss": 0.8123, + "step": 2210 + }, + { + "epoch": 1.6145454545454545, + "grad_norm": 0.5763828754425049, + "learning_rate": 1.9910011248593928e-06, + "loss": 0.7833, + "step": 2220 + }, + { + "epoch": 1.6218181818181818, + "grad_norm": 0.582817792892456, + "learning_rate": 1.9535058117735283e-06, + "loss": 0.8126, + "step": 2230 + }, + { + "epoch": 1.6290909090909091, + "grad_norm": 0.5699591040611267, + "learning_rate": 1.9160104986876642e-06, + "loss": 0.8128, + "step": 2240 + }, + { + "epoch": 1.6363636363636362, + "grad_norm": 0.5779479742050171, + "learning_rate": 1.8785151856018e-06, + "loss": 0.8121, + "step": 2250 + }, + { + "epoch": 1.6436363636363636, + "grad_norm": 0.5711218118667603, + "learning_rate": 1.8410198725159355e-06, + "loss": 0.8032, + "step": 2260 + }, + { + "epoch": 1.6509090909090909, + "grad_norm": 0.6018957495689392, + "learning_rate": 1.8035245594300713e-06, + "loss": 0.8125, + "step": 2270 + }, + { + "epoch": 1.6581818181818182, + "grad_norm": 0.570153534412384, + "learning_rate": 1.7660292463442072e-06, + "loss": 0.8143, + "step": 2280 + }, + { + "epoch": 1.6654545454545455, + "grad_norm": 0.5871094465255737, + "learning_rate": 1.728533933258343e-06, + "loss": 0.8179, + "step": 2290 + }, + { + "epoch": 1.6727272727272728, + "grad_norm": 0.5894734859466553, + "learning_rate": 1.6910386201724785e-06, + "loss": 0.8015, + "step": 2300 + }, + { + "epoch": 1.6800000000000002, + "grad_norm": 0.6005243062973022, + "learning_rate": 1.6535433070866144e-06, + "loss": 0.8009, + "step": 2310 + }, + { + "epoch": 1.6872727272727273, + "grad_norm": 0.5786961317062378, + "learning_rate": 1.6160479940007502e-06, + "loss": 0.82, + "step": 2320 + }, + { + "epoch": 1.6945454545454546, + "grad_norm": 0.5606324672698975, + "learning_rate": 1.5785526809148857e-06, + "loss": 0.8074, + "step": 2330 + }, + { + "epoch": 1.7018181818181817, + "grad_norm": 0.5757468938827515, + "learning_rate": 1.5410573678290214e-06, + "loss": 0.7997, + "step": 2340 + }, + { + "epoch": 1.709090909090909, + "grad_norm": 0.5801776051521301, + "learning_rate": 1.5035620547431574e-06, + "loss": 0.8159, + "step": 2350 + }, + { + "epoch": 1.7163636363636363, + "grad_norm": 0.561937689781189, + "learning_rate": 1.4660667416572929e-06, + "loss": 0.8088, + "step": 2360 + }, + { + "epoch": 1.7236363636363636, + "grad_norm": 0.581520140171051, + "learning_rate": 1.4285714285714286e-06, + "loss": 0.8137, + "step": 2370 + }, + { + "epoch": 1.730909090909091, + "grad_norm": 0.5766869783401489, + "learning_rate": 1.3910761154855646e-06, + "loss": 0.8115, + "step": 2380 + }, + { + "epoch": 1.7381818181818183, + "grad_norm": 0.5791675448417664, + "learning_rate": 1.3535808023997e-06, + "loss": 0.7978, + "step": 2390 + }, + { + "epoch": 1.7454545454545456, + "grad_norm": 0.5874686241149902, + "learning_rate": 1.3160854893138358e-06, + "loss": 0.8226, + "step": 2400 + }, + { + "epoch": 1.7527272727272727, + "grad_norm": 0.5620043277740479, + "learning_rate": 1.2785901762279716e-06, + "loss": 0.7917, + "step": 2410 + }, + { + "epoch": 1.76, + "grad_norm": 0.563328206539154, + "learning_rate": 1.2410948631421073e-06, + "loss": 0.8061, + "step": 2420 + }, + { + "epoch": 1.767272727272727, + "grad_norm": 0.5629045963287354, + "learning_rate": 1.203599550056243e-06, + "loss": 0.8104, + "step": 2430 + }, + { + "epoch": 1.7745454545454544, + "grad_norm": 0.5813615918159485, + "learning_rate": 1.1661042369703788e-06, + "loss": 0.8108, + "step": 2440 + }, + { + "epoch": 1.7818181818181817, + "grad_norm": 0.5680747628211975, + "learning_rate": 1.1286089238845145e-06, + "loss": 0.8117, + "step": 2450 + }, + { + "epoch": 1.789090909090909, + "grad_norm": 0.5629953742027283, + "learning_rate": 1.0911136107986503e-06, + "loss": 0.7866, + "step": 2460 + }, + { + "epoch": 1.7963636363636364, + "grad_norm": 0.5827091932296753, + "learning_rate": 1.053618297712786e-06, + "loss": 0.7988, + "step": 2470 + }, + { + "epoch": 1.8036363636363637, + "grad_norm": 0.5806558728218079, + "learning_rate": 1.0161229846269217e-06, + "loss": 0.8169, + "step": 2480 + }, + { + "epoch": 1.810909090909091, + "grad_norm": 0.5832482576370239, + "learning_rate": 9.786276715410575e-07, + "loss": 0.8205, + "step": 2490 + }, + { + "epoch": 1.8181818181818183, + "grad_norm": 0.5685613751411438, + "learning_rate": 9.411323584551932e-07, + "loss": 0.8071, + "step": 2500 + }, + { + "epoch": 1.8254545454545454, + "grad_norm": 0.6105993390083313, + "learning_rate": 9.03637045369329e-07, + "loss": 0.8097, + "step": 2510 + }, + { + "epoch": 1.8327272727272728, + "grad_norm": 0.5787394046783447, + "learning_rate": 8.661417322834646e-07, + "loss": 0.8085, + "step": 2520 + }, + { + "epoch": 1.8399999999999999, + "grad_norm": 0.5817368030548096, + "learning_rate": 8.286464191976004e-07, + "loss": 0.7993, + "step": 2530 + }, + { + "epoch": 1.8472727272727272, + "grad_norm": 0.5391492247581482, + "learning_rate": 7.911511061117361e-07, + "loss": 0.8174, + "step": 2540 + }, + { + "epoch": 1.8545454545454545, + "grad_norm": 0.5950977206230164, + "learning_rate": 7.536557930258719e-07, + "loss": 0.8269, + "step": 2550 + }, + { + "epoch": 1.8618181818181818, + "grad_norm": 0.5649157166481018, + "learning_rate": 7.161604799400075e-07, + "loss": 0.8101, + "step": 2560 + }, + { + "epoch": 1.8690909090909091, + "grad_norm": 0.5596181154251099, + "learning_rate": 6.786651668541433e-07, + "loss": 0.8058, + "step": 2570 + }, + { + "epoch": 1.8763636363636365, + "grad_norm": 0.5372329354286194, + "learning_rate": 6.411698537682789e-07, + "loss": 0.8121, + "step": 2580 + }, + { + "epoch": 1.8836363636363638, + "grad_norm": 0.5739951133728027, + "learning_rate": 6.036745406824148e-07, + "loss": 0.8098, + "step": 2590 + }, + { + "epoch": 1.8909090909090909, + "grad_norm": 0.5760049223899841, + "learning_rate": 5.661792275965505e-07, + "loss": 0.813, + "step": 2600 + }, + { + "epoch": 1.8981818181818182, + "grad_norm": 0.5617313981056213, + "learning_rate": 5.286839145106862e-07, + "loss": 0.8019, + "step": 2610 + }, + { + "epoch": 1.9054545454545453, + "grad_norm": 0.5732679963111877, + "learning_rate": 4.91188601424822e-07, + "loss": 0.8049, + "step": 2620 + }, + { + "epoch": 1.9127272727272726, + "grad_norm": 0.5599384307861328, + "learning_rate": 4.536932883389577e-07, + "loss": 0.7894, + "step": 2630 + }, + { + "epoch": 1.92, + "grad_norm": 0.5590060353279114, + "learning_rate": 4.161979752530934e-07, + "loss": 0.8021, + "step": 2640 + }, + { + "epoch": 1.9272727272727272, + "grad_norm": 0.5524166822433472, + "learning_rate": 3.787026621672291e-07, + "loss": 0.7941, + "step": 2650 + }, + { + "epoch": 1.9345454545454546, + "grad_norm": 0.5761271715164185, + "learning_rate": 3.4120734908136486e-07, + "loss": 0.8243, + "step": 2660 + }, + { + "epoch": 1.9418181818181819, + "grad_norm": 0.5809080600738525, + "learning_rate": 3.037120359955006e-07, + "loss": 0.804, + "step": 2670 + }, + { + "epoch": 1.9490909090909092, + "grad_norm": 0.5772132873535156, + "learning_rate": 2.6621672290963634e-07, + "loss": 0.8194, + "step": 2680 + }, + { + "epoch": 1.9563636363636365, + "grad_norm": 0.5759870409965515, + "learning_rate": 2.2872140982377205e-07, + "loss": 0.822, + "step": 2690 + }, + { + "epoch": 1.9636363636363636, + "grad_norm": 0.5843390226364136, + "learning_rate": 1.9122609673790776e-07, + "loss": 0.8049, + "step": 2700 + }, + { + "epoch": 1.970909090909091, + "grad_norm": 0.5843693614006042, + "learning_rate": 1.537307836520435e-07, + "loss": 0.8169, + "step": 2710 + }, + { + "epoch": 1.978181818181818, + "grad_norm": 0.5962398052215576, + "learning_rate": 1.1623547056617925e-07, + "loss": 0.8102, + "step": 2720 + }, + { + "epoch": 1.9854545454545454, + "grad_norm": 0.5839393734931946, + "learning_rate": 7.874015748031497e-08, + "loss": 0.7998, + "step": 2730 + }, + { + "epoch": 1.9927272727272727, + "grad_norm": 0.5290048122406006, + "learning_rate": 4.1244844394450697e-08, + "loss": 0.7923, + "step": 2740 + }, + { + "epoch": 2.0, + "grad_norm": 0.581894040107727, + "learning_rate": 3.7495313085864276e-09, + "loss": 0.8008, + "step": 2750 + }, + { + "epoch": 2.0, + "step": 2750, + "total_flos": 8.418445674676224e+18, + "train_loss": 0.8579012258703058, + "train_runtime": 27047.1098, + "train_samples_per_second": 13.014, + "train_steps_per_second": 0.102 + } + ], + "logging_steps": 10, + "max_steps": 2750, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 1000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 8.418445674676224e+18, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}