From a107bbde8759ea879b547fb0e57ae59ae6c5fe23 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 20 Jul 2026 20:51:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: aadityabuilds/qwen2-5-coder-7b-kernelbook-sft-equal-tokens Source: Original Platform --- .gitattributes | 36 ++ README.md | 74 ++++ chat_template.jinja | 54 +++ config.json | 61 ++++ generation_config.json | 13 + model.safetensors | 3 + run_metadata.json | 137 +++++++ tokenizer.json | 3 + tokenizer_config.json | 30 ++ trainer_state.json | 811 +++++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 11 files changed, 1225 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 run_metadata.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..89b111b --- /dev/null +++ b/README.md @@ -0,0 +1,74 @@ +--- +library_name: transformers +license: apache-2.0 +base_model: Qwen/Qwen2.5-Coder-7B-Instruct +tags: +- triton +- kernelbook +- code-generation +- sft +- trl +- text-generation +datasets: +- custom +language: +- en +pipeline_tag: text-generation +--- + +# Qwen2.5-Coder-7B KernelBook SFT (equal tokens) + +**Supervised Fine-Tuning (SFT)** checkpoint of [Qwen/Qwen2.5-Coder-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct), post-trained on the **KernelBook** Triton kernel dataset. + +This repo is the **equal-exposure** SFT checkpoint (`checkpoint-350`, ~1.06 epochs) selected to match SDFT's one-epoch training for fair comparison. + +## Method + +This model was trained with **SFT** using TRL's `SFTTrainer`: standard next-token prediction on chat-formatted prompt → Triton completion pairs, with completion-only loss (prompt tokens masked). Training used DeepSpeed ZeRO-3 and bf16 on Modal. + +## Dataset + +- **KernelBook** — PyTorch module prompts paired with reference Triton kernels +- Deduplicated, filtered to completions ≤4096 tokens, repo-stratified 80/10/10 split +- Stopped at **checkpoint-350** (~1.06 epochs) for parity with the SDFT run + +## Intended use + +Generate Triton GPU kernels from PyTorch-style module descriptions. Best for KernelBook-style conversion prompts; not evaluated as a general-purpose chat or reasoning model. + +## Quick start + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_id = "aadityabuilds/qwen2-5-coder-7b-kernelbook-sft-equal-tokens" +tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained( + model_id, torch_dtype="auto", device_map="auto", trust_remote_code=True +) + +messages = [ + { + "role": "user", + "content": "Convert the following PyTorch code to an equivalent Triton kernel...", + } +] +prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) +inputs = tokenizer(prompt, return_tensors="pt").to(model.device) +outputs = model.generate(**inputs, max_new_tokens=1200, do_sample=False) +print(tokenizer.decode(outputs[0][inputs["input_ids"].shape[1] :], skip_special_tokens=True)) +``` + +## Training summary + +| Setting | Value | +|---------|-------| +| Base model | Qwen2.5-Coder-7B-Instruct | +| Method | SFT (TRL `SFTTrainer`, completion-only NLL) | +| Checkpoint | `checkpoint-350` (~1.06 epochs) | +| Hardware | 4× H100 (Modal) | +| Parallelism | DeepSpeed ZeRO-3, bf16 | + +## Limitations + +Specialized for KernelBook Triton codegen. May show reduced performance on general coding, math, and knowledge benchmarks compared to the base instruct model. diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..1edb7e2 --- /dev/null +++ b/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.9.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..21d3d9d --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.9.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..65864b6 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c06e0a0ac4463a59fc386d61666ad0325173157e6764809c38f80fb43c84f79 +size 15231272152 diff --git a/run_metadata.json b/run_metadata.json new file mode 100644 index 0000000..917b99e --- /dev/null +++ b/run_metadata.json @@ -0,0 +1,137 @@ +{ + "cli_args": { + "attn_implementation": "eager", + "auto_resume": false, + "bf16": true, + "cache_dir": "/cache", + "data_dir": "/workspace/data/kernelbook", + "dataloader_num_workers": 4, + "deepspeed": "/workspace/configs/deepspeed/zero3_bf16.json", + "dry_run": false, + "effective_batch_size": 32, + "eval_packing": false, + "eval_steps": 50, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": true, + "hub_model_id": "aadityabuilds/qwen2-5-coder-7b-kernelbook-sft", + "learning_rate": 2e-05, + "logging_steps": 5, + "loss_type": "nll", + "max_eval_samples": null, + "max_grad_norm": 1.0, + "max_length": 8192, + "max_steps": -1, + "max_train_samples": null, + "model": "/cache/local-models/sft/qwen2-5-coder-7b-instruct", + "num_train_epochs": 3.0, + "output_dir": "/__modal/volumes/vo-qWxmkR9prkx4LKrjcfqOmD/modal-sft-qwen2-5-coder-7b-kernelbook-final-nopack-eager", + "output_root": "/outputs", + "packing": false, + "packing_strategy": "bfd", + "per_device_eval_batch_size": 1, + "per_device_train_batch_size": 1, + "push_to_hub": true, + "report_to": "wandb", + "resume_from_checkpoint": null, + "run_name": "modal-sft-qwen2-5-coder-7b-kernelbook-final-nopack-eager", + "save_steps": 50, + "save_total_limit": 10, + "seed": 42, + "target_global_batch_size": 32, + "wandb_entity": null, + "wandb_mode": "online", + "wandb_project": "triton-sdft", + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 4 + }, + "data_dir": "/workspace/data/kernelbook", + "effective_batch_size": 32, + "manifest": { + "config": { + "created_at": "2026-05-27T05:16:47.175016+00:00", + "dataset_id": "GPUMODE/KernelBook", + "max_output_tokens": 4096, + "max_seq_length": 8192, + "model": "Qwen/Qwen2.5-Coder-7B-Instruct", + "output_dir": "data/kernelbook", + "seed": 42, + "test_ratio": 0.1, + "train_ratio": 0.8, + "val_ratio": 0.1 + }, + "counts": { + "after_dedup": 15203, + "after_empty_filter": 18162, + "after_output_length_filter": 13267, + "loaded": 18162, + "test": 1360, + "train": 10578, + "validation": 1329 + }, + "sdft_trainer": { + "eval_dataset": "data/kernelbook/text/sdft/validation", + "sdft_config_hints": { + "generate_from_teacher": true, + "max_completion_length": 4096, + "max_prompt_length": 4096 + }, + "test_dataset": "data/kernelbook/text/sdft/test", + "train_dataset": "data/kernelbook/text/sdft/train" + }, + "sft_trainer": { + "eval_dataset": "data/kernelbook/tokenized/Qwen2.5-Coder-7B-Instruct/validation", + "eval_packing": false, + "packing": true, + "requires_columns": [ + "input_ids", + "completion_mask" + ], + "sft_config": { + "completion_only_loss": true, + "eval_packing": false, + "max_length": 8192, + "packing": true + }, + "test_dataset": "data/kernelbook/tokenized/Qwen2.5-Coder-7B-Instruct/test", + "train_dataset": "data/kernelbook/tokenized/Qwen2.5-Coder-7B-Instruct/train" + }, + "token_stats": { + "test": { + "count": 1360.0, + "max": 6072.0, + "min": 519.0, + "p50": 1742.5, + "p90": 3393.1000000000013, + "p95": 4133.1, + "p99": 4980.400000000003, + "truncated_fraction": 0.0 + }, + "train": { + "count": 10578.0, + "max": 7026.0, + "min": 517.0, + "p50": 1781.5, + "p90": 3559.0, + "p95": 4168.299999999999, + "p99": 4932.459999999999, + "truncated_fraction": 0.0 + }, + "validation": { + "count": 1329.0, + "max": 7012.0, + "min": 519.0, + "p50": 1787.0, + "p90": 3371.2, + "p95": 3914.3999999999996, + "p99": 4647.0, + "truncated_fraction": 0.0 + } + } + }, + "method": "sft", + "model": "/cache/local-models/sft/qwen2-5-coder-7b-instruct", + "output_dir": "/__modal/volumes/vo-qWxmkR9prkx4LKrjcfqOmD/modal-sft-qwen2-5-coder-7b-kernelbook-final-nopack-eager", + "run_name": "modal-sft-qwen2-5-coder-7b-kernelbook-final-nopack-eager", + "world_size": 4 +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..7498a30 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 32768, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..1170ed7 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,811 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0574669187145558, + "eval_steps": 50, + "global_step": 350, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.442822265625, + "epoch": 0.015122873345935728, + "grad_norm": 2.669071628071661, + "learning_rate": 2.666666666666667e-06, + "loss": 0.5928983211517334, + "mean_token_accuracy": 0.8548422813415527, + "num_tokens": 325432.0, + "step": 5 + }, + { + "entropy": 0.3935546875, + "epoch": 0.030245746691871456, + "grad_norm": 1.634140660130009, + "learning_rate": 6e-06, + "loss": 0.3988723993301392, + "mean_token_accuracy": 0.8959574371576309, + "num_tokens": 653193.0, + "step": 10 + }, + { + "entropy": 0.2187255859375, + "epoch": 0.045368620037807186, + "grad_norm": 0.9021532274129692, + "learning_rate": 9.333333333333334e-06, + "loss": 0.19831626415252684, + "mean_token_accuracy": 0.9389900401234627, + "num_tokens": 964453.0, + "step": 15 + }, + { + "entropy": 0.10172119140625, + "epoch": 0.06049149338374291, + "grad_norm": 0.4697352375821627, + "learning_rate": 1.2666666666666667e-05, + "loss": 0.10886554718017578, + "mean_token_accuracy": 0.9637670248746872, + "num_tokens": 1307238.0, + "step": 20 + }, + { + "entropy": 0.078204345703125, + "epoch": 0.07561436672967864, + "grad_norm": 0.39345906256866536, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.0776131808757782, + "mean_token_accuracy": 0.9740302637219429, + "num_tokens": 1615672.0, + "step": 25 + }, + { + "entropy": 0.05927734375, + "epoch": 0.09073724007561437, + "grad_norm": 0.35679792282054484, + "learning_rate": 1.9333333333333333e-05, + "loss": 0.06322811841964722, + "mean_token_accuracy": 0.9787174344062806, + "num_tokens": 1935122.0, + "step": 30 + }, + { + "entropy": 0.055340576171875, + "epoch": 0.10586011342155009, + "grad_norm": 0.28127188289574473, + "learning_rate": 1.99991486051959e-05, + "loss": 0.055406326055526735, + "mean_token_accuracy": 0.9811019197106361, + "num_tokens": 2246001.0, + "step": 35 + }, + { + "entropy": 0.050018310546875, + "epoch": 0.12098298676748583, + "grad_norm": 0.23485165839069644, + "learning_rate": 1.9995690062269985e-05, + "loss": 0.05159962177276611, + "mean_token_accuracy": 0.9823019146919251, + "num_tokens": 2572728.0, + "step": 40 + }, + { + "entropy": 0.05009765625, + "epoch": 0.13610586011342155, + "grad_norm": 0.3286661753766386, + "learning_rate": 1.9989572078514722e-05, + "loss": 0.04924889206886292, + "mean_token_accuracy": 0.983111384510994, + "num_tokens": 2899222.0, + "step": 45 + }, + { + "entropy": 0.03935394287109375, + "epoch": 0.15122873345935728, + "grad_norm": 0.250685968179852, + "learning_rate": 1.998079628167323e-05, + "loss": 0.041659212112426756, + "mean_token_accuracy": 0.9857470989227295, + "num_tokens": 3199913.0, + "step": 50 + }, + { + "epoch": 0.15122873345935728, + "eval_entropy": 0.040476572764170424, + "eval_loss": 0.0391058586537838, + "eval_mean_token_accuracy": 0.986595441092242, + "eval_num_tokens": 3199913.0, + "eval_runtime": 94.4655, + "eval_samples_per_second": 14.069, + "eval_steps_per_second": 3.525, + "step": 50 + }, + { + "entropy": 0.038671875, + "epoch": 0.166351606805293, + "grad_norm": 0.3054203398912923, + "learning_rate": 1.9969365006623072e-05, + "loss": 0.03826351165771484, + "mean_token_accuracy": 0.9868962660431861, + "num_tokens": 3519539.0, + "step": 55 + }, + { + "entropy": 0.035321044921875, + "epoch": 0.18147448015122875, + "grad_norm": 0.3848999837215887, + "learning_rate": 1.9955281294755034e-05, + "loss": 0.03624363541603089, + "mean_token_accuracy": 0.9876649409532547, + "num_tokens": 3846310.0, + "step": 60 + }, + { + "entropy": 0.035552978515625, + "epoch": 0.19659735349716445, + "grad_norm": 0.2606280391924725, + "learning_rate": 1.9938548893163924e-05, + "loss": 0.03628517687320709, + "mean_token_accuracy": 0.9875744208693504, + "num_tokens": 4173916.0, + "step": 65 + }, + { + "entropy": 0.0336944580078125, + "epoch": 0.21172022684310018, + "grad_norm": 0.24422090914230693, + "learning_rate": 1.9919172253651637e-05, + "loss": 0.033189314603805545, + "mean_token_accuracy": 0.9885250389575958, + "num_tokens": 4494452.0, + "step": 70 + }, + { + "entropy": 0.02899322509765625, + "epoch": 0.22684310018903592, + "grad_norm": 0.19378251727544818, + "learning_rate": 1.9897156531542708e-05, + "loss": 0.029927724599838258, + "mean_token_accuracy": 0.9899556040763855, + "num_tokens": 4813996.0, + "step": 75 + }, + { + "entropy": 0.0284515380859375, + "epoch": 0.24196597353497165, + "grad_norm": 0.2664077757245733, + "learning_rate": 1.98725075843127e-05, + "loss": 0.029463762044906618, + "mean_token_accuracy": 0.9898741751909256, + "num_tokens": 5154020.0, + "step": 80 + }, + { + "entropy": 0.0311767578125, + "epoch": 0.2570888468809074, + "grad_norm": 0.22537314543102535, + "learning_rate": 1.9845231970029774e-05, + "loss": 0.03209342658519745, + "mean_token_accuracy": 0.9891638234257698, + "num_tokens": 5484973.0, + "step": 85 + }, + { + "entropy": 0.0276885986328125, + "epoch": 0.2722117202268431, + "grad_norm": 0.2077524527319594, + "learning_rate": 1.9815336945609834e-05, + "loss": 0.029284730553627014, + "mean_token_accuracy": 0.9901994511485099, + "num_tokens": 5811817.0, + "step": 90 + }, + { + "entropy": 0.0260345458984375, + "epoch": 0.28733459357277885, + "grad_norm": 0.31080264218901027, + "learning_rate": 1.9782830464885788e-05, + "loss": 0.02565096616744995, + "mean_token_accuracy": 0.991170009970665, + "num_tokens": 6118979.0, + "step": 95 + }, + { + "entropy": 0.024970245361328126, + "epoch": 0.30245746691871456, + "grad_norm": 0.21180271207392207, + "learning_rate": 1.974772117649135e-05, + "loss": 0.024624763429164885, + "mean_token_accuracy": 0.9913630470633507, + "num_tokens": 6457003.0, + "step": 100 + }, + { + "epoch": 0.30245746691871456, + "eval_entropy": 0.02398535009618994, + "eval_loss": 0.025832440704107285, + "eval_mean_token_accuracy": 0.9910635029947436, + "eval_num_tokens": 6457003.0, + "eval_runtime": 92.826, + "eval_samples_per_second": 14.317, + "eval_steps_per_second": 3.587, + "step": 100 + }, + { + "entropy": 0.0255767822265625, + "epoch": 0.31758034026465026, + "grad_norm": 0.19039718090172789, + "learning_rate": 1.9710018421559995e-05, + "loss": 0.026742279529571533, + "mean_token_accuracy": 0.9909975379705429, + "num_tokens": 6768352.0, + "step": 105 + }, + { + "entropy": 0.02334747314453125, + "epoch": 0.332703213610586, + "grad_norm": 0.15632220147486883, + "learning_rate": 1.966973223123967e-05, + "loss": 0.021733593940734864, + "mean_token_accuracy": 0.9925331950187684, + "num_tokens": 7086098.0, + "step": 110 + }, + { + "entropy": 0.02210235595703125, + "epoch": 0.34782608695652173, + "grad_norm": 0.2458450644732953, + "learning_rate": 1.9626873324023915e-05, + "loss": 0.02539382576942444, + "mean_token_accuracy": 0.9917543068528175, + "num_tokens": 7397835.0, + "step": 115 + }, + { + "entropy": 0.022796630859375, + "epoch": 0.3629489603024575, + "grad_norm": 0.511944627378666, + "learning_rate": 1.9581453102900127e-05, + "loss": 0.022371816635131835, + "mean_token_accuracy": 0.9924163356423378, + "num_tokens": 7720248.0, + "step": 120 + }, + { + "entropy": 0.02347259521484375, + "epoch": 0.3780718336483932, + "grad_norm": 0.22819135667809418, + "learning_rate": 1.953348365231568e-05, + "loss": 0.02383442670106888, + "mean_token_accuracy": 0.9916202604770661, + "num_tokens": 8046850.0, + "step": 125 + }, + { + "entropy": 0.02361907958984375, + "epoch": 0.3931947069943289, + "grad_norm": 0.23351254559117343, + "learning_rate": 1.9482977734962753e-05, + "loss": 0.0238445445895195, + "mean_token_accuracy": 0.9918097272515297, + "num_tokens": 8390721.0, + "step": 130 + }, + { + "entropy": 0.0227020263671875, + "epoch": 0.40831758034026466, + "grad_norm": 0.18760418206446414, + "learning_rate": 1.9429948788382726e-05, + "loss": 0.022852468490600585, + "mean_token_accuracy": 0.9926092982292175, + "num_tokens": 8741140.0, + "step": 135 + }, + { + "entropy": 0.02464141845703125, + "epoch": 0.42344045368620037, + "grad_norm": 0.17532160128342728, + "learning_rate": 1.937441092139097e-05, + "loss": 0.025189298391342162, + "mean_token_accuracy": 0.9915472865104675, + "num_tokens": 9066217.0, + "step": 140 + }, + { + "entropy": 0.022135162353515626, + "epoch": 0.43856332703213613, + "grad_norm": 0.2898150379760738, + "learning_rate": 1.93163789103231e-05, + "loss": 0.02323928475379944, + "mean_token_accuracy": 0.9923686638474465, + "num_tokens": 9398518.0, + "step": 145 + }, + { + "entropy": 0.022913360595703126, + "epoch": 0.45368620037807184, + "grad_norm": 0.20882908714005557, + "learning_rate": 1.9255868195103596e-05, + "loss": 0.023580947518348695, + "mean_token_accuracy": 0.9922584399580956, + "num_tokens": 9722146.0, + "step": 150 + }, + { + "epoch": 0.45368620037807184, + "eval_entropy": 0.019961612002627627, + "eval_loss": 0.021232564002275467, + "eval_mean_token_accuracy": 0.9926312886200868, + "eval_num_tokens": 9722146.0, + "eval_runtime": 92.911, + "eval_samples_per_second": 14.304, + "eval_steps_per_second": 3.584, + "step": 150 + }, + { + "entropy": 0.02125396728515625, + "epoch": 0.46880907372400754, + "grad_norm": 0.16611123987930837, + "learning_rate": 1.9192894875137864e-05, + "loss": 0.022398817539215087, + "mean_token_accuracy": 0.9927845776081086, + "num_tokens": 10051130.0, + "step": 155 + }, + { + "entropy": 0.01784820556640625, + "epoch": 0.4839319470699433, + "grad_norm": 0.1705996270647505, + "learning_rate": 1.9127475705028864e-05, + "loss": 0.020223259925842285, + "mean_token_accuracy": 0.9935442849993705, + "num_tokens": 10364936.0, + "step": 160 + }, + { + "entropy": 0.02181854248046875, + "epoch": 0.499054820415879, + "grad_norm": 0.18993585672995894, + "learning_rate": 1.9059628090119392e-05, + "loss": 0.02131263315677643, + "mean_token_accuracy": 0.9929479479789733, + "num_tokens": 10684934.0, + "step": 165 + }, + { + "entropy": 0.01970977783203125, + "epoch": 0.5141776937618148, + "grad_norm": 0.23862681974692412, + "learning_rate": 1.898937008186123e-05, + "loss": 0.021003079414367676, + "mean_token_accuracy": 0.992812205851078, + "num_tokens": 11016640.0, + "step": 170 + }, + { + "entropy": 0.02065277099609375, + "epoch": 0.5293005671077504, + "grad_norm": 0.22580792158388535, + "learning_rate": 1.8916720373012425e-05, + "loss": 0.020279140770435335, + "mean_token_accuracy": 0.9934262841939926, + "num_tokens": 11335375.0, + "step": 175 + }, + { + "entropy": 0.02077484130859375, + "epoch": 0.5444234404536862, + "grad_norm": 0.18177690286836462, + "learning_rate": 1.8841698292663887e-05, + "loss": 0.021297402679920197, + "mean_token_accuracy": 0.9927030056715012, + "num_tokens": 11669483.0, + "step": 180 + }, + { + "entropy": 0.020357513427734376, + "epoch": 0.5595463137996219, + "grad_norm": 0.18951806144385833, + "learning_rate": 1.876432380109673e-05, + "loss": 0.02147246301174164, + "mean_token_accuracy": 0.9929309383034706, + "num_tokens": 11987399.0, + "step": 185 + }, + { + "entropy": 0.021063995361328126, + "epoch": 0.5746691871455577, + "grad_norm": 0.28328991051134833, + "learning_rate": 1.8684617484471662e-05, + "loss": 0.021729469299316406, + "mean_token_accuracy": 0.9926604524254798, + "num_tokens": 12321769.0, + "step": 190 + }, + { + "entropy": 0.016484832763671874, + "epoch": 0.5897920604914934, + "grad_norm": 0.2604043117050105, + "learning_rate": 1.860260054935186e-05, + "loss": 0.016648142039775847, + "mean_token_accuracy": 0.9943771168589592, + "num_tokens": 12637459.0, + "step": 195 + }, + { + "entropy": 0.016230010986328126, + "epoch": 0.6049149338374291, + "grad_norm": 0.1518215594544113, + "learning_rate": 1.8518294817060777e-05, + "loss": 0.016636769473552703, + "mean_token_accuracy": 0.9944247990846634, + "num_tokens": 12976497.0, + "step": 200 + }, + { + "epoch": 0.6049149338374291, + "eval_entropy": 0.01848311324019332, + "eval_loss": 0.018460111692547798, + "eval_mean_token_accuracy": 0.9936752308596362, + "eval_num_tokens": 12976497.0, + "eval_runtime": 93.7884, + "eval_samples_per_second": 14.17, + "eval_steps_per_second": 3.551, + "step": 200 + }, + { + "entropy": 0.01815185546875, + "epoch": 0.6200378071833649, + "grad_norm": 0.2097083339362261, + "learning_rate": 1.8431722717876383e-05, + "loss": 0.01749354600906372, + "mean_token_accuracy": 0.993867652118206, + "num_tokens": 13302976.0, + "step": 205 + }, + { + "entropy": 0.01755523681640625, + "epoch": 0.6351606805293005, + "grad_norm": 0.17477493752672238, + "learning_rate": 1.834290728506339e-05, + "loss": 0.018500179052352905, + "mean_token_accuracy": 0.9936169162392616, + "num_tokens": 13627997.0, + "step": 210 + }, + { + "entropy": 0.018619537353515625, + "epoch": 0.6502835538752363, + "grad_norm": 0.1969863482217428, + "learning_rate": 1.8251872148745075e-05, + "loss": 0.017858733236789704, + "mean_token_accuracy": 0.9939072072505951, + "num_tokens": 13945104.0, + "step": 215 + }, + { + "entropy": 0.019687652587890625, + "epoch": 0.665406427221172, + "grad_norm": 0.2554011428793856, + "learning_rate": 1.815864152961624e-05, + "loss": 0.02146708369255066, + "mean_token_accuracy": 0.9928086891770362, + "num_tokens": 14279010.0, + "step": 220 + }, + { + "entropy": 0.0164215087890625, + "epoch": 0.6805293005671077, + "grad_norm": 0.22053987762264646, + "learning_rate": 1.806324023249912e-05, + "loss": 0.017443592846393585, + "mean_token_accuracy": 0.9943347543478012, + "num_tokens": 14608053.0, + "step": 225 + }, + { + "entropy": 0.01520233154296875, + "epoch": 0.6956521739130435, + "grad_norm": 0.15495690088956657, + "learning_rate": 1.7965693639743816e-05, + "loss": 0.015465393662452698, + "mean_token_accuracy": 0.9950109854340553, + "num_tokens": 14925691.0, + "step": 230 + }, + { + "entropy": 0.0169342041015625, + "epoch": 0.7107750472589792, + "grad_norm": 0.23516476733142996, + "learning_rate": 1.786602770447513e-05, + "loss": 0.017490310966968535, + "mean_token_accuracy": 0.9939287096261978, + "num_tokens": 15247510.0, + "step": 235 + }, + { + "entropy": 0.015985107421875, + "epoch": 0.725897920604915, + "grad_norm": 0.13394070957421397, + "learning_rate": 1.776426894368746e-05, + "loss": 0.015802830457687378, + "mean_token_accuracy": 0.9947777166962624, + "num_tokens": 15560305.0, + "step": 240 + }, + { + "entropy": 0.014200592041015625, + "epoch": 0.7410207939508506, + "grad_norm": 0.19432987534155574, + "learning_rate": 1.766044443118978e-05, + "loss": 0.014611494541168214, + "mean_token_accuracy": 0.9948656380176544, + "num_tokens": 15898787.0, + "step": 245 + }, + { + "entropy": 0.0140716552734375, + "epoch": 0.7561436672967864, + "grad_norm": 0.16144920711925453, + "learning_rate": 1.7554581790402372e-05, + "loss": 0.015314009785652161, + "mean_token_accuracy": 0.9950292810797692, + "num_tokens": 16236000.0, + "step": 250 + }, + { + "epoch": 0.7561436672967864, + "eval_entropy": 0.01619310350389452, + "eval_loss": 0.016467994078993797, + "eval_mean_token_accuracy": 0.9943410491084194, + "eval_num_tokens": 16236000.0, + "eval_runtime": 92.8452, + "eval_samples_per_second": 14.314, + "eval_steps_per_second": 3.587, + "step": 250 + }, + { + "entropy": 0.017685699462890624, + "epoch": 0.7712665406427222, + "grad_norm": 0.17754228327893673, + "learning_rate": 1.7446709187007383e-05, + "loss": 0.018420988321304323, + "mean_token_accuracy": 0.9938467770814896, + "num_tokens": 16568742.0, + "step": 255 + }, + { + "entropy": 0.01598701477050781, + "epoch": 0.7863894139886578, + "grad_norm": 0.1753647354705704, + "learning_rate": 1.7336855321455126e-05, + "loss": 0.016123867034912108, + "mean_token_accuracy": 0.9946757212281228, + "num_tokens": 16886171.0, + "step": 260 + }, + { + "entropy": 0.01281585693359375, + "epoch": 0.8015122873345936, + "grad_norm": 0.1694346707825758, + "learning_rate": 1.7225049421328024e-05, + "loss": 0.01325436681509018, + "mean_token_accuracy": 0.9954081103205681, + "num_tokens": 17213518.0, + "step": 265 + }, + { + "entropy": 0.015825653076171876, + "epoch": 0.8166351606805293, + "grad_norm": 0.12320756121826888, + "learning_rate": 1.7111321233564406e-05, + "loss": 0.016265182197093962, + "mean_token_accuracy": 0.9945767849683762, + "num_tokens": 17526983.0, + "step": 270 + }, + { + "entropy": 0.01460418701171875, + "epoch": 0.831758034026465, + "grad_norm": 0.15708115348818796, + "learning_rate": 1.699570101654404e-05, + "loss": 0.013886567950248719, + "mean_token_accuracy": 0.9951759412884712, + "num_tokens": 17844226.0, + "step": 275 + }, + { + "entropy": 0.012836456298828125, + "epoch": 0.8468809073724007, + "grad_norm": 0.143701427710118, + "learning_rate": 1.687821953203765e-05, + "loss": 0.01477896273136139, + "mean_token_accuracy": 0.9952674239873887, + "num_tokens": 18154251.0, + "step": 280 + }, + { + "entropy": 0.013329315185546874, + "epoch": 0.8620037807183365, + "grad_norm": 0.14730660194978837, + "learning_rate": 1.6758908037022446e-05, + "loss": 0.01359080672264099, + "mean_token_accuracy": 0.9955309063196183, + "num_tokens": 18468667.0, + "step": 285 + }, + { + "entropy": 0.01543426513671875, + "epoch": 0.8771266540642723, + "grad_norm": 0.13900308540287734, + "learning_rate": 1.6637798275365976e-05, + "loss": 0.016150069236755372, + "mean_token_accuracy": 0.9946491599082947, + "num_tokens": 18806646.0, + "step": 290 + }, + { + "entropy": 0.012978363037109374, + "epoch": 0.8922495274102079, + "grad_norm": 0.13732124684095226, + "learning_rate": 1.651492246938034e-05, + "loss": 0.012651418149471284, + "mean_token_accuracy": 0.995641553401947, + "num_tokens": 19132756.0, + "step": 295 + }, + { + "entropy": 0.015012359619140625, + "epoch": 0.9073724007561437, + "grad_norm": 0.13977045253544546, + "learning_rate": 1.6390313311249212e-05, + "loss": 0.01540960967540741, + "mean_token_accuracy": 0.9949710816144943, + "num_tokens": 19465713.0, + "step": 300 + }, + { + "epoch": 0.9073724007561437, + "eval_entropy": 0.014344957139756944, + "eval_loss": 0.015235255472362041, + "eval_mean_token_accuracy": 0.9947822013416806, + "eval_num_tokens": 19465713.0, + "eval_runtime": 92.3526, + "eval_samples_per_second": 14.391, + "eval_steps_per_second": 3.606, + "step": 300 + }, + { + "entropy": 0.013709640502929688, + "epoch": 0.9224952741020794, + "grad_norm": 0.19119314364034928, + "learning_rate": 1.6264003954329762e-05, + "loss": 0.01464797556400299, + "mean_token_accuracy": 0.9951745182275772, + "num_tokens": 19791857.0, + "step": 305 + }, + { + "entropy": 0.0113555908203125, + "epoch": 0.9376181474480151, + "grad_norm": 0.13382137381631254, + "learning_rate": 1.613602800433194e-05, + "loss": 0.012380270659923554, + "mean_token_accuracy": 0.9959253877401352, + "num_tokens": 20121726.0, + "step": 310 + }, + { + "entropy": 0.013879013061523438, + "epoch": 0.9527410207939508, + "grad_norm": 0.1786900745985722, + "learning_rate": 1.600641951037736e-05, + "loss": 0.013191039860248565, + "mean_token_accuracy": 0.9955014079809189, + "num_tokens": 20434281.0, + "step": 315 + }, + { + "entropy": 0.014342498779296876, + "epoch": 0.9678638941398866, + "grad_norm": 0.20383676268015227, + "learning_rate": 1.5875212955940255e-05, + "loss": 0.01611756980419159, + "mean_token_accuracy": 0.9947072401642799, + "num_tokens": 20753074.0, + "step": 320 + }, + { + "entropy": 0.013257980346679688, + "epoch": 0.9829867674858223, + "grad_norm": 0.1364426748258384, + "learning_rate": 1.574244324967283e-05, + "loss": 0.013307541608810425, + "mean_token_accuracy": 0.9954897597432136, + "num_tokens": 21086031.0, + "step": 325 + }, + { + "entropy": 0.0136016845703125, + "epoch": 0.998109640831758, + "grad_norm": 0.11139044118229276, + "learning_rate": 1.5608145716117507e-05, + "loss": 0.013101503252983093, + "mean_token_accuracy": 0.9954297497868538, + "num_tokens": 21385002.0, + "step": 330 + }, + { + "entropy": 0.010610631994298986, + "epoch": 1.0120982986767486, + "grad_norm": 0.11937254210200125, + "learning_rate": 1.547235608630856e-05, + "loss": 0.009768091142177582, + "mean_token_accuracy": 0.9965261527009912, + "num_tokens": 21651447.0, + "step": 335 + }, + { + "entropy": 0.008671951293945313, + "epoch": 1.0272211720226843, + "grad_norm": 0.12948932598689103, + "learning_rate": 1.5335110488265497e-05, + "loss": 0.00888235867023468, + "mean_token_accuracy": 0.996855866909027, + "num_tokens": 21981761.0, + "step": 340 + }, + { + "entropy": 0.009595108032226563, + "epoch": 1.0423440453686201, + "grad_norm": 0.21649437613862998, + "learning_rate": 1.5196445437380941e-05, + "loss": 0.010289055854082107, + "mean_token_accuracy": 0.9964772433042526, + "num_tokens": 22320903.0, + "step": 345 + }, + { + "entropy": 0.00907135009765625, + "epoch": 1.0574669187145558, + "grad_norm": 0.12728450692524118, + "learning_rate": 1.5056397826705356e-05, + "loss": 0.008513091504573822, + "mean_token_accuracy": 0.9971326634287834, + "num_tokens": 22652825.0, + "step": 350 + }, + { + "epoch": 1.0574669187145558, + "eval_entropy": 0.011664244505736205, + "eval_loss": 0.013880621641874313, + "eval_mean_token_accuracy": 0.9952635595032403, + "eval_num_tokens": 22652825.0, + "eval_runtime": 91.6429, + "eval_samples_per_second": 14.502, + "eval_steps_per_second": 3.634, + "step": 350 + } + ], + "logging_steps": 5, + "max_steps": 993, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 45311893700608.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..41bfd46 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d1e6776123b74114e6853f6976af58526ec3f3c5ab44ef83ef85b38acf0358d +size 7761