commit 5311039f38b2f99f72e1799fcac28a07f6fe7d5c Author: ModelHub XC Date: Fri Jun 26 22:54:12 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: mlfoundations-dev/bespoke_stratos_0.3k Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..62da7f4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,56 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text + + +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text + +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +tokenizer.json filter=lfs diff=lfs merge=lfs -text +model-00001-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +merges.txt filter=lfs diff=lfs merge=lfs -text +training_args.bin filter=lfs diff=lfs merge=lfs -text +model-00002-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +vocab.json filter=lfs diff=lfs merge=lfs -text +model-00004-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +model-00003-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..ef03bb1 --- /dev/null +++ b/README.md @@ -0,0 +1,61 @@ +--- +library_name: transformers +license: apache-2.0 +base_model: Qwen/Qwen2.5-7B-Instruct +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: bespoke_stratos_0.3k + results: [] +--- + + + +# bespoke_stratos_0.3k + +This model is a fine-tuned version of [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) on the mlfoundations-dev/bespoke_stratos_0.3k dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 1e-05 +- train_batch_size: 1 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 8 +- total_train_batch_size: 32 +- total_eval_batch_size: 32 +- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 13.0 + +### Training results + + + +### Framework versions + +- Transformers 4.46.1 +- Pytorch 2.6.0+cu124 +- Datasets 3.1.0 +- Tokenizers 0.20.3 diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..482ced4 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,24 @@ +{ + "": 151658, + "": 151657, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..1b79375 --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 12.30379746835443, + "total_flos": 4.902218749915955e+16, + "train_loss": 0.26980868625080484, + "train_runtime": 2812.7387, + "train_samples_per_second": 1.46, + "train_steps_per_second": 0.042 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..6c52571 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "_name_or_path": "Qwen/Qwen2.5-7B-Instruct", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000.0, + "sliding_window": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.46.1", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/configs.yaml b/configs.yaml new file mode 100644 index 0000000..3930106 --- /dev/null +++ b/configs.yaml @@ -0,0 +1,37 @@ +assistant_tag: gpt +bf16: true +content_tag: value +cutoff_len: 16384 +dataloader_num_workers: 4 +dataloader_persistent_workers: true +dataloader_pin_memory: true +dataset: mlfoundations-dev/bespoke_stratos_0.3k +dataset_dir: ONLINE +ddp_timeout: 180000000 +deepspeed: dcft/train/zero3.json +do_train: true +enable_liger_kernel: true +finetuning_type: full +global_batch_size: 32 +gradient_accumulation_steps: 8 +hub_model_id: mlfoundations-dev/bespoke_stratos_0.3k +learning_rate: 1.0e-05 +logging_steps: 1 +lr_scheduler_type: cosine +messages: conversations +model_name_or_path: Qwen/Qwen2.5-7B-Instruct +num_train_epochs: 13.0 +output_dir: /data/cat/ws/ryma833h-dcft/checkpoints/bespoke_stratos_0.3k +overwrite_cache: true +per_device_train_batch_size: 1 +plot_loss: true +preprocessing_num_workers: 16 +push_to_db: true +push_to_hub: true +report_to: wandb +role_tag: from +save_strategy: epoch +stage: sft +template: qwen25 +user_tag: human +warmup_ratio: 0.1 diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..a753841 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,14 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "4.46.1" +} diff --git a/merges.txt b/merges.txt new file mode 100644 index 0000000..80c1a19 --- /dev/null +++ b/merges.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5 +size 1671853 diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..0116440 --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3d9fcdf01a87cbb3f61149e98641014bff4bf229b611173d0c6a58e5375b6e3 +size 4877660776 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..3854b45 --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:881ec43116bda77d6d2f062ea21098042bb225ab2c857c236650b684b5316f85 +size 4932751008 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..fb3011e --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c83b48be686e8f7ea49f0b7b29062fda20ff281e2313e517597b44c42184568 +size 4330865200 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..70e3234 --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41dab64cc81b3b312d37498ad82ced156164342229bf25c4b42d5474e3eb06ce +size 1089994880 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..6ca5084 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,346 @@ +{ + "metadata": { + "total_size": 15231233024 + }, + "weight_map": { + "lm_head.weight": "model-00004-of-00004.safetensors", + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00003-of-00004.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..17305b3 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..51ebb3b --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa +size 11421896 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..b84f53a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,208 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- else %}\n {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- message.content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n", + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..1b79375 --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 12.30379746835443, + "total_flos": 4.902218749915955e+16, + "train_loss": 0.26980868625080484, + "train_runtime": 2812.7387, + "train_samples_per_second": 1.46, + "train_steps_per_second": 0.042 +} \ No newline at end of file diff --git a/trainer_log.jsonl b/trainer_log.jsonl new file mode 100644 index 0000000..5fcfa98 --- /dev/null +++ b/trainer_log.jsonl @@ -0,0 +1,118 @@ +{"current_steps": 1, "total_steps": 117, "loss": 0.9281, "lr": 8.333333333333333e-07, "epoch": 0.10126582278481013, "percentage": 0.85, "elapsed_time": "0:00:19", "remaining_time": "0:37:25"} +{"current_steps": 2, "total_steps": 117, "loss": 0.9108, "lr": 1.6666666666666667e-06, "epoch": 0.20253164556962025, "percentage": 1.71, "elapsed_time": "0:00:41", "remaining_time": "0:39:35"} +{"current_steps": 3, "total_steps": 117, "loss": 0.8874, "lr": 2.5e-06, "epoch": 0.3037974683544304, "percentage": 2.56, "elapsed_time": "0:00:57", "remaining_time": "0:36:39"} +{"current_steps": 4, "total_steps": 117, "loss": 0.9102, "lr": 3.3333333333333333e-06, "epoch": 0.4050632911392405, "percentage": 3.42, "elapsed_time": "0:01:16", "remaining_time": "0:35:51"} +{"current_steps": 5, "total_steps": 117, "loss": 0.8248, "lr": 4.166666666666667e-06, "epoch": 0.5063291139240507, "percentage": 4.27, "elapsed_time": "0:01:31", "remaining_time": "0:34:07"} +{"current_steps": 6, "total_steps": 117, "loss": 0.8383, "lr": 5e-06, "epoch": 0.6075949367088608, "percentage": 5.13, "elapsed_time": "0:01:50", "remaining_time": "0:34:07"} +{"current_steps": 7, "total_steps": 117, "loss": 0.7354, "lr": 5.833333333333334e-06, "epoch": 0.7088607594936709, "percentage": 5.98, "elapsed_time": "0:02:08", "remaining_time": "0:33:44"} +{"current_steps": 8, "total_steps": 117, "loss": 0.7928, "lr": 6.666666666666667e-06, "epoch": 0.810126582278481, "percentage": 6.84, "elapsed_time": "0:02:26", "remaining_time": "0:33:11"} +{"current_steps": 9, "total_steps": 117, "loss": 0.7744, "lr": 7.500000000000001e-06, "epoch": 0.9113924050632911, "percentage": 7.69, "elapsed_time": "0:02:37", "remaining_time": "0:31:35"} +{"current_steps": 10, "total_steps": 117, "loss": 0.7524, "lr": 8.333333333333334e-06, "epoch": 1.0506329113924051, "percentage": 8.55, "elapsed_time": "0:03:43", "remaining_time": "0:39:46"} +{"current_steps": 11, "total_steps": 117, "loss": 0.6639, "lr": 9.166666666666666e-06, "epoch": 1.1518987341772151, "percentage": 9.4, "elapsed_time": "0:03:55", "remaining_time": "0:37:47"} +{"current_steps": 12, "total_steps": 117, "loss": 0.6936, "lr": 1e-05, "epoch": 1.2531645569620253, "percentage": 10.26, "elapsed_time": "0:04:11", "remaining_time": "0:36:40"} +{"current_steps": 13, "total_steps": 117, "loss": 0.6682, "lr": 9.997762161417517e-06, "epoch": 1.3544303797468356, "percentage": 11.11, "elapsed_time": "0:04:39", "remaining_time": "0:37:13"} +{"current_steps": 14, "total_steps": 117, "loss": 0.5577, "lr": 9.991050648838676e-06, "epoch": 1.4556962025316456, "percentage": 11.97, "elapsed_time": "0:04:55", "remaining_time": "0:36:13"} +{"current_steps": 15, "total_steps": 117, "loss": 0.5829, "lr": 9.979871469976197e-06, "epoch": 1.5569620253164556, "percentage": 12.82, "elapsed_time": "0:05:16", "remaining_time": "0:35:53"} +{"current_steps": 16, "total_steps": 117, "loss": 0.6041, "lr": 9.964234631709188e-06, "epoch": 1.6582278481012658, "percentage": 13.68, "elapsed_time": "0:05:33", "remaining_time": "0:35:05"} +{"current_steps": 17, "total_steps": 117, "loss": 0.6291, "lr": 9.944154131125643e-06, "epoch": 1.759493670886076, "percentage": 14.53, "elapsed_time": "0:05:47", "remaining_time": "0:34:05"} +{"current_steps": 18, "total_steps": 117, "loss": 0.6081, "lr": 9.91964794299315e-06, "epoch": 1.8607594936708862, "percentage": 15.38, "elapsed_time": "0:06:02", "remaining_time": "0:33:16"} +{"current_steps": 19, "total_steps": 117, "loss": 0.5927, "lr": 9.890738003669029e-06, "epoch": 1.9620253164556962, "percentage": 16.24, "elapsed_time": "0:06:24", "remaining_time": "0:33:01"} +{"current_steps": 20, "total_steps": 117, "loss": 0.5682, "lr": 9.857450191464337e-06, "epoch": 2.1012658227848102, "percentage": 17.09, "elapsed_time": "0:07:18", "remaining_time": "0:35:26"} +{"current_steps": 21, "total_steps": 117, "loss": 0.5403, "lr": 9.819814303479268e-06, "epoch": 2.2025316455696204, "percentage": 17.95, "elapsed_time": "0:07:33", "remaining_time": "0:34:32"} +{"current_steps": 22, "total_steps": 117, "loss": 0.4855, "lr": 9.777864028930705e-06, "epoch": 2.3037974683544302, "percentage": 18.8, "elapsed_time": "0:07:48", "remaining_time": "0:33:42"} +{"current_steps": 23, "total_steps": 117, "loss": 0.5031, "lr": 9.731636918995821e-06, "epoch": 2.4050632911392404, "percentage": 19.66, "elapsed_time": "0:08:07", "remaining_time": "0:33:13"} +{"current_steps": 24, "total_steps": 117, "loss": 0.4571, "lr": 9.681174353198687e-06, "epoch": 2.5063291139240507, "percentage": 20.51, "elapsed_time": "0:08:25", "remaining_time": "0:32:37"} +{"current_steps": 25, "total_steps": 117, "loss": 0.4672, "lr": 9.626521502369984e-06, "epoch": 2.607594936708861, "percentage": 21.37, "elapsed_time": "0:08:37", "remaining_time": "0:31:45"} +{"current_steps": 26, "total_steps": 117, "loss": 0.4384, "lr": 9.567727288213005e-06, "epoch": 2.708860759493671, "percentage": 22.22, "elapsed_time": "0:08:57", "remaining_time": "0:31:22"} +{"current_steps": 27, "total_steps": 117, "loss": 0.3985, "lr": 9.504844339512096e-06, "epoch": 2.810126582278481, "percentage": 23.08, "elapsed_time": "0:09:18", "remaining_time": "0:31:00"} +{"current_steps": 28, "total_steps": 117, "loss": 0.4792, "lr": 9.437928945022772e-06, "epoch": 2.911392405063291, "percentage": 23.93, "elapsed_time": "0:09:38", "remaining_time": "0:30:38"} +{"current_steps": 29, "total_steps": 117, "loss": 0.362, "lr": 9.36704100308565e-06, "epoch": 3.050632911392405, "percentage": 24.79, "elapsed_time": "0:10:49", "remaining_time": "0:32:50"} +{"current_steps": 30, "total_steps": 117, "loss": 0.3723, "lr": 9.292243968009332e-06, "epoch": 3.151898734177215, "percentage": 25.64, "elapsed_time": "0:11:09", "remaining_time": "0:32:22"} +{"current_steps": 31, "total_steps": 117, "loss": 0.3369, "lr": 9.213604793270196e-06, "epoch": 3.2531645569620253, "percentage": 26.5, "elapsed_time": "0:11:31", "remaining_time": "0:31:57"} +{"current_steps": 32, "total_steps": 117, "loss": 0.3604, "lr": 9.131193871579975e-06, "epoch": 3.3544303797468356, "percentage": 27.35, "elapsed_time": "0:11:54", "remaining_time": "0:31:37"} +{"current_steps": 33, "total_steps": 117, "loss": 0.3631, "lr": 9.045084971874738e-06, "epoch": 3.4556962025316453, "percentage": 28.21, "elapsed_time": "0:12:12", "remaining_time": "0:31:04"} +{"current_steps": 34, "total_steps": 117, "loss": 0.3553, "lr": 8.955355173281709e-06, "epoch": 3.5569620253164556, "percentage": 29.06, "elapsed_time": "0:12:28", "remaining_time": "0:30:28"} +{"current_steps": 35, "total_steps": 117, "loss": 0.4085, "lr": 8.862084796122998e-06, "epoch": 3.6582278481012658, "percentage": 29.91, "elapsed_time": "0:12:47", "remaining_time": "0:29:58"} +{"current_steps": 36, "total_steps": 117, "loss": 0.2889, "lr": 8.765357330018056e-06, "epoch": 3.759493670886076, "percentage": 30.77, "elapsed_time": "0:13:03", "remaining_time": "0:29:23"} +{"current_steps": 37, "total_steps": 117, "loss": 0.2859, "lr": 8.665259359149132e-06, "epoch": 3.8607594936708862, "percentage": 31.62, "elapsed_time": "0:13:15", "remaining_time": "0:28:39"} +{"current_steps": 38, "total_steps": 117, "loss": 0.36, "lr": 8.561880484756726e-06, "epoch": 3.962025316455696, "percentage": 32.48, "elapsed_time": "0:13:29", "remaining_time": "0:28:02"} +{"current_steps": 39, "total_steps": 117, "loss": 0.284, "lr": 8.455313244934324e-06, "epoch": 4.10126582278481, "percentage": 33.33, "elapsed_time": "0:14:41", "remaining_time": "0:29:23"} +{"current_steps": 40, "total_steps": 117, "loss": 0.2657, "lr": 8.345653031794292e-06, "epoch": 4.2025316455696204, "percentage": 34.19, "elapsed_time": "0:14:58", "remaining_time": "0:28:49"} +{"current_steps": 41, "total_steps": 117, "loss": 0.2237, "lr": 8.232998006078998e-06, "epoch": 4.30379746835443, "percentage": 35.04, "elapsed_time": "0:15:11", "remaining_time": "0:28:10"} +{"current_steps": 42, "total_steps": 117, "loss": 0.2345, "lr": 8.117449009293668e-06, "epoch": 4.405063291139241, "percentage": 35.9, "elapsed_time": "0:15:26", "remaining_time": "0:27:34"} +{"current_steps": 43, "total_steps": 117, "loss": 0.2498, "lr": 7.99910947343957e-06, "epoch": 4.506329113924051, "percentage": 36.75, "elapsed_time": "0:15:42", "remaining_time": "0:27:01"} +{"current_steps": 44, "total_steps": 117, "loss": 0.2405, "lr": 7.87808532842837e-06, "epoch": 4.6075949367088604, "percentage": 37.61, "elapsed_time": "0:15:56", "remaining_time": "0:26:26"} +{"current_steps": 45, "total_steps": 117, "loss": 0.2913, "lr": 7.754484907260513e-06, "epoch": 4.708860759493671, "percentage": 38.46, "elapsed_time": "0:16:19", "remaining_time": "0:26:07"} +{"current_steps": 46, "total_steps": 117, "loss": 0.286, "lr": 7.628418849052523e-06, "epoch": 4.810126582278481, "percentage": 39.32, "elapsed_time": "0:16:38", "remaining_time": "0:25:41"} +{"current_steps": 47, "total_steps": 117, "loss": 0.2364, "lr": 7.500000000000001e-06, "epoch": 4.911392405063291, "percentage": 40.17, "elapsed_time": "0:16:59", "remaining_time": "0:25:18"} +{"current_steps": 48, "total_steps": 117, "loss": 0.2076, "lr": 7.369343312364994e-06, "epoch": 5.050632911392405, "percentage": 41.03, "elapsed_time": "0:18:13", "remaining_time": "0:26:11"} +{"current_steps": 49, "total_steps": 117, "loss": 0.1667, "lr": 7.236565741578163e-06, "epoch": 5.151898734177215, "percentage": 41.88, "elapsed_time": "0:18:34", "remaining_time": "0:25:46"} +{"current_steps": 50, "total_steps": 117, "loss": 0.1684, "lr": 7.101786141547829e-06, "epoch": 5.253164556962025, "percentage": 42.74, "elapsed_time": "0:18:51", "remaining_time": "0:25:16"} +{"current_steps": 51, "total_steps": 117, "loss": 0.2039, "lr": 6.965125158269619e-06, "epoch": 5.3544303797468356, "percentage": 43.59, "elapsed_time": "0:19:10", "remaining_time": "0:24:48"} +{"current_steps": 52, "total_steps": 117, "loss": 0.1794, "lr": 6.8267051218319766e-06, "epoch": 5.455696202531645, "percentage": 44.44, "elapsed_time": "0:19:24", "remaining_time": "0:24:16"} +{"current_steps": 53, "total_steps": 117, "loss": 0.1529, "lr": 6.686649936914151e-06, "epoch": 5.556962025316456, "percentage": 45.3, "elapsed_time": "0:19:35", "remaining_time": "0:23:39"} +{"current_steps": 54, "total_steps": 117, "loss": 0.1956, "lr": 6.545084971874738e-06, "epoch": 5.658227848101266, "percentage": 46.15, "elapsed_time": "0:19:55", "remaining_time": "0:23:15"} +{"current_steps": 55, "total_steps": 117, "loss": 0.1958, "lr": 6.402136946530014e-06, "epoch": 5.759493670886076, "percentage": 47.01, "elapsed_time": "0:20:14", "remaining_time": "0:22:48"} +{"current_steps": 56, "total_steps": 117, "loss": 0.217, "lr": 6.257933818722544e-06, "epoch": 5.860759493670886, "percentage": 47.86, "elapsed_time": "0:20:35", "remaining_time": "0:22:26"} +{"current_steps": 57, "total_steps": 117, "loss": 0.1946, "lr": 6.112604669781572e-06, "epoch": 5.962025316455696, "percentage": 48.72, "elapsed_time": "0:20:58", "remaining_time": "0:22:04"} +{"current_steps": 58, "total_steps": 117, "loss": 0.1701, "lr": 5.9662795889777666e-06, "epoch": 6.10126582278481, "percentage": 49.57, "elapsed_time": "0:22:14", "remaining_time": "0:22:37"} +{"current_steps": 59, "total_steps": 117, "loss": 0.1426, "lr": 5.819089557075689e-06, "epoch": 6.2025316455696204, "percentage": 50.43, "elapsed_time": "0:22:30", "remaining_time": "0:22:07"} +{"current_steps": 60, "total_steps": 117, "loss": 0.1184, "lr": 5.671166329088278e-06, "epoch": 6.30379746835443, "percentage": 51.28, "elapsed_time": "0:22:42", "remaining_time": "0:21:34"} +{"current_steps": 61, "total_steps": 117, "loss": 0.1978, "lr": 5.522642316338268e-06, "epoch": 6.405063291139241, "percentage": 52.14, "elapsed_time": "0:22:58", "remaining_time": "0:21:05"} +{"current_steps": 62, "total_steps": 117, "loss": 0.1521, "lr": 5.373650467932122e-06, "epoch": 6.506329113924051, "percentage": 52.99, "elapsed_time": "0:23:14", "remaining_time": "0:20:36"} +{"current_steps": 63, "total_steps": 117, "loss": 0.1556, "lr": 5.224324151752575e-06, "epoch": 6.6075949367088604, "percentage": 53.85, "elapsed_time": "0:23:34", "remaining_time": "0:20:12"} +{"current_steps": 64, "total_steps": 117, "loss": 0.1134, "lr": 5.074797035076319e-06, "epoch": 6.708860759493671, "percentage": 54.7, "elapsed_time": "0:23:47", "remaining_time": "0:19:42"} +{"current_steps": 65, "total_steps": 117, "loss": 0.1085, "lr": 4.9252029649236835e-06, "epoch": 6.810126582278481, "percentage": 55.56, "elapsed_time": "0:23:58", "remaining_time": "0:19:10"} +{"current_steps": 66, "total_steps": 117, "loss": 0.1296, "lr": 4.775675848247427e-06, "epoch": 6.911392405063291, "percentage": 56.41, "elapsed_time": "0:24:13", "remaining_time": "0:18:42"} +{"current_steps": 67, "total_steps": 117, "loss": 0.1223, "lr": 4.626349532067879e-06, "epoch": 7.050632911392405, "percentage": 57.26, "elapsed_time": "0:25:26", "remaining_time": "0:18:59"} +{"current_steps": 68, "total_steps": 117, "loss": 0.0976, "lr": 4.477357683661734e-06, "epoch": 7.151898734177215, "percentage": 58.12, "elapsed_time": "0:25:41", "remaining_time": "0:18:30"} +{"current_steps": 69, "total_steps": 117, "loss": 0.1576, "lr": 4.3288336709117246e-06, "epoch": 7.253164556962025, "percentage": 58.97, "elapsed_time": "0:26:02", "remaining_time": "0:18:06"} +{"current_steps": 70, "total_steps": 117, "loss": 0.1002, "lr": 4.180910442924312e-06, "epoch": 7.3544303797468356, "percentage": 59.83, "elapsed_time": "0:26:17", "remaining_time": "0:17:39"} +{"current_steps": 71, "total_steps": 117, "loss": 0.0634, "lr": 4.033720411022235e-06, "epoch": 7.455696202531645, "percentage": 60.68, "elapsed_time": "0:26:28", "remaining_time": "0:17:09"} +{"current_steps": 72, "total_steps": 117, "loss": 0.1324, "lr": 3.887395330218429e-06, "epoch": 7.556962025316456, "percentage": 61.54, "elapsed_time": "0:26:50", "remaining_time": "0:16:46"} +{"current_steps": 73, "total_steps": 117, "loss": 0.0783, "lr": 3.7420661812774577e-06, "epoch": 7.658227848101266, "percentage": 62.39, "elapsed_time": "0:27:05", "remaining_time": "0:16:19"} +{"current_steps": 74, "total_steps": 117, "loss": 0.1258, "lr": 3.5978630534699873e-06, "epoch": 7.759493670886076, "percentage": 63.25, "elapsed_time": "0:27:28", "remaining_time": "0:15:58"} +{"current_steps": 75, "total_steps": 117, "loss": 0.1357, "lr": 3.4549150281252635e-06, "epoch": 7.860759493670886, "percentage": 64.1, "elapsed_time": "0:27:47", "remaining_time": "0:15:33"} +{"current_steps": 76, "total_steps": 117, "loss": 0.1221, "lr": 3.3133500630858507e-06, "epoch": 7.962025316455696, "percentage": 64.96, "elapsed_time": "0:28:02", "remaining_time": "0:15:07"} +{"current_steps": 77, "total_steps": 117, "loss": 0.0895, "lr": 3.173294878168025e-06, "epoch": 8.10126582278481, "percentage": 65.81, "elapsed_time": "0:29:27", "remaining_time": "0:15:18"} +{"current_steps": 78, "total_steps": 117, "loss": 0.1119, "lr": 3.0348748417303826e-06, "epoch": 8.20253164556962, "percentage": 66.67, "elapsed_time": "0:29:47", "remaining_time": "0:14:53"} +{"current_steps": 79, "total_steps": 117, "loss": 0.0546, "lr": 2.8982138584521734e-06, "epoch": 8.30379746835443, "percentage": 67.52, "elapsed_time": "0:29:58", "remaining_time": "0:14:25"} +{"current_steps": 80, "total_steps": 117, "loss": 0.0701, "lr": 2.7634342584218364e-06, "epoch": 8.405063291139241, "percentage": 68.38, "elapsed_time": "0:30:15", "remaining_time": "0:13:59"} +{"current_steps": 81, "total_steps": 117, "loss": 0.088, "lr": 2.6306566876350072e-06, "epoch": 8.50632911392405, "percentage": 69.23, "elapsed_time": "0:30:32", "remaining_time": "0:13:34"} +{"current_steps": 82, "total_steps": 117, "loss": 0.0989, "lr": 2.5000000000000015e-06, "epoch": 8.60759493670886, "percentage": 70.09, "elapsed_time": "0:30:51", "remaining_time": "0:13:10"} +{"current_steps": 83, "total_steps": 117, "loss": 0.1138, "lr": 2.371581150947476e-06, "epoch": 8.708860759493671, "percentage": 70.94, "elapsed_time": "0:31:11", "remaining_time": "0:12:46"} +{"current_steps": 84, "total_steps": 117, "loss": 0.0778, "lr": 2.245515092739488e-06, "epoch": 8.810126582278482, "percentage": 71.79, "elapsed_time": "0:31:26", "remaining_time": "0:12:21"} +{"current_steps": 85, "total_steps": 117, "loss": 0.1197, "lr": 2.1219146715716332e-06, "epoch": 8.91139240506329, "percentage": 72.65, "elapsed_time": "0:31:44", "remaining_time": "0:11:57"} +{"current_steps": 86, "total_steps": 117, "loss": 0.0645, "lr": 2.0008905265604316e-06, "epoch": 9.050632911392405, "percentage": 73.5, "elapsed_time": "0:32:52", "remaining_time": "0:11:50"} +{"current_steps": 87, "total_steps": 117, "loss": 0.0707, "lr": 1.8825509907063328e-06, "epoch": 9.151898734177216, "percentage": 74.36, "elapsed_time": "0:33:08", "remaining_time": "0:11:25"} +{"current_steps": 88, "total_steps": 117, "loss": 0.0592, "lr": 1.7670019939210025e-06, "epoch": 9.253164556962025, "percentage": 75.21, "elapsed_time": "0:33:22", "remaining_time": "0:10:59"} +{"current_steps": 89, "total_steps": 117, "loss": 0.107, "lr": 1.6543469682057105e-06, "epoch": 9.354430379746836, "percentage": 76.07, "elapsed_time": "0:33:41", "remaining_time": "0:10:36"} +{"current_steps": 90, "total_steps": 117, "loss": 0.0515, "lr": 1.544686755065677e-06, "epoch": 9.455696202531646, "percentage": 76.92, "elapsed_time": "0:33:55", "remaining_time": "0:10:10"} +{"current_steps": 91, "total_steps": 117, "loss": 0.1044, "lr": 1.438119515243277e-06, "epoch": 9.556962025316455, "percentage": 77.78, "elapsed_time": "0:34:20", "remaining_time": "0:09:48"} +{"current_steps": 92, "total_steps": 117, "loss": 0.0741, "lr": 1.3347406408508695e-06, "epoch": 9.658227848101266, "percentage": 78.63, "elapsed_time": "0:34:37", "remaining_time": "0:09:24"} +{"current_steps": 93, "total_steps": 117, "loss": 0.1096, "lr": 1.234642669981946e-06, "epoch": 9.759493670886076, "percentage": 79.49, "elapsed_time": "0:34:56", "remaining_time": "0:09:00"} +{"current_steps": 94, "total_steps": 117, "loss": 0.0738, "lr": 1.137915203877003e-06, "epoch": 9.860759493670885, "percentage": 80.34, "elapsed_time": "0:35:11", "remaining_time": "0:08:36"} +{"current_steps": 95, "total_steps": 117, "loss": 0.0798, "lr": 1.044644826718295e-06, "epoch": 9.962025316455696, "percentage": 81.2, "elapsed_time": "0:35:34", "remaining_time": "0:08:14"} +{"current_steps": 96, "total_steps": 117, "loss": 0.0772, "lr": 9.549150281252633e-07, "epoch": 10.10126582278481, "percentage": 82.05, "elapsed_time": "0:36:54", "remaining_time": "0:08:04"} +{"current_steps": 97, "total_steps": 117, "loss": 0.0706, "lr": 8.688061284200266e-07, "epoch": 10.20253164556962, "percentage": 82.91, "elapsed_time": "0:37:09", "remaining_time": "0:07:39"} +{"current_steps": 98, "total_steps": 117, "loss": 0.0854, "lr": 7.863952067298042e-07, "epoch": 10.30379746835443, "percentage": 83.76, "elapsed_time": "0:37:33", "remaining_time": "0:07:16"} +{"current_steps": 99, "total_steps": 117, "loss": 0.0463, "lr": 7.077560319906696e-07, "epoch": 10.405063291139241, "percentage": 84.62, "elapsed_time": "0:37:46", "remaining_time": "0:06:52"} +{"current_steps": 100, "total_steps": 117, "loss": 0.0642, "lr": 6.329589969143518e-07, "epoch": 10.50632911392405, "percentage": 85.47, "elapsed_time": "0:37:58", "remaining_time": "0:06:27"} +{"current_steps": 101, "total_steps": 117, "loss": 0.0462, "lr": 5.620710549772295e-07, "epoch": 10.60759493670886, "percentage": 86.32, "elapsed_time": "0:38:13", "remaining_time": "0:06:03"} +{"current_steps": 102, "total_steps": 117, "loss": 0.1179, "lr": 4.951556604879049e-07, "epoch": 10.708860759493671, "percentage": 87.18, "elapsed_time": "0:38:36", "remaining_time": "0:05:40"} +{"current_steps": 103, "total_steps": 117, "loss": 0.0675, "lr": 4.322727117869951e-07, "epoch": 10.810126582278482, "percentage": 88.03, "elapsed_time": "0:38:53", "remaining_time": "0:05:17"} +{"current_steps": 104, "total_steps": 117, "loss": 0.0693, "lr": 3.734784976300165e-07, "epoch": 10.91139240506329, "percentage": 88.89, "elapsed_time": "0:39:12", "remaining_time": "0:04:54"} +{"current_steps": 105, "total_steps": 117, "loss": 0.0957, "lr": 3.18825646801314e-07, "epoch": 11.050632911392405, "percentage": 89.74, "elapsed_time": "0:40:35", "remaining_time": "0:04:38"} +{"current_steps": 106, "total_steps": 117, "loss": 0.041, "lr": 2.6836308100417874e-07, "epoch": 11.151898734177216, "percentage": 90.6, "elapsed_time": "0:40:49", "remaining_time": "0:04:14"} +{"current_steps": 107, "total_steps": 117, "loss": 0.0564, "lr": 2.2213597106929608e-07, "epoch": 11.253164556962025, "percentage": 91.45, "elapsed_time": "0:41:02", "remaining_time": "0:03:50"} +{"current_steps": 108, "total_steps": 117, "loss": 0.0804, "lr": 1.801856965207338e-07, "epoch": 11.354430379746836, "percentage": 92.31, "elapsed_time": "0:41:18", "remaining_time": "0:03:26"} +{"current_steps": 109, "total_steps": 117, "loss": 0.041, "lr": 1.4254980853566248e-07, "epoch": 11.455696202531646, "percentage": 93.16, "elapsed_time": "0:41:33", "remaining_time": "0:03:03"} +{"current_steps": 110, "total_steps": 117, "loss": 0.0903, "lr": 1.0926199633097156e-07, "epoch": 11.556962025316455, "percentage": 94.02, "elapsed_time": "0:41:52", "remaining_time": "0:02:39"} +{"current_steps": 111, "total_steps": 117, "loss": 0.0721, "lr": 8.035205700685167e-08, "epoch": 11.658227848101266, "percentage": 94.87, "elapsed_time": "0:42:09", "remaining_time": "0:02:16"} +{"current_steps": 112, "total_steps": 117, "loss": 0.0657, "lr": 5.584586887435739e-08, "epoch": 11.759493670886076, "percentage": 95.73, "elapsed_time": "0:42:30", "remaining_time": "0:01:53"} +{"current_steps": 113, "total_steps": 117, "loss": 0.078, "lr": 3.576536829081323e-08, "epoch": 11.860759493670885, "percentage": 96.58, "elapsed_time": "0:42:48", "remaining_time": "0:01:30"} +{"current_steps": 114, "total_steps": 117, "loss": 0.0652, "lr": 2.012853002380466e-08, "epoch": 11.962025316455696, "percentage": 97.44, "elapsed_time": "0:43:05", "remaining_time": "0:01:08"} +{"current_steps": 115, "total_steps": 117, "loss": 0.0842, "lr": 8.949351161324227e-09, "epoch": 12.10126582278481, "percentage": 98.29, "elapsed_time": "0:44:14", "remaining_time": "0:00:46"} +{"current_steps": 116, "total_steps": 117, "loss": 0.0994, "lr": 2.237838582483387e-09, "epoch": 12.20253164556962, "percentage": 99.15, "elapsed_time": "0:44:41", "remaining_time": "0:00:23"} +{"current_steps": 117, "total_steps": 117, "loss": 0.0314, "lr": 0.0, "epoch": 12.30379746835443, "percentage": 100.0, "elapsed_time": "0:44:56", "remaining_time": "0:00:00"} +{"current_steps": 117, "total_steps": 117, "epoch": 12.30379746835443, "percentage": 100.0, "elapsed_time": "0:46:50", "remaining_time": "0:00:00"} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..f721c72 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,861 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 12.30379746835443, + "eval_steps": 500, + "global_step": 117, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.10126582278481013, + "grad_norm": 6.43825208149568, + "learning_rate": 8.333333333333333e-07, + "loss": 0.9281, + "step": 1 + }, + { + "epoch": 0.20253164556962025, + "grad_norm": 6.559153069315341, + "learning_rate": 1.6666666666666667e-06, + "loss": 0.9108, + "step": 2 + }, + { + "epoch": 0.3037974683544304, + "grad_norm": 7.045196702684219, + "learning_rate": 2.5e-06, + "loss": 0.8874, + "step": 3 + }, + { + "epoch": 0.4050632911392405, + "grad_norm": 6.450857710098491, + "learning_rate": 3.3333333333333333e-06, + "loss": 0.9102, + "step": 4 + }, + { + "epoch": 0.5063291139240507, + "grad_norm": 5.274518665853932, + "learning_rate": 4.166666666666667e-06, + "loss": 0.8248, + "step": 5 + }, + { + "epoch": 0.6075949367088608, + "grad_norm": 3.2648170762097477, + "learning_rate": 5e-06, + "loss": 0.8383, + "step": 6 + }, + { + "epoch": 0.7088607594936709, + "grad_norm": 2.5880252500398844, + "learning_rate": 5.833333333333334e-06, + "loss": 0.7354, + "step": 7 + }, + { + "epoch": 0.810126582278481, + "grad_norm": 3.845109483810076, + "learning_rate": 6.666666666666667e-06, + "loss": 0.7928, + "step": 8 + }, + { + "epoch": 0.9113924050632911, + "grad_norm": 4.109400755644465, + "learning_rate": 7.500000000000001e-06, + "loss": 0.7744, + "step": 9 + }, + { + "epoch": 1.0506329113924051, + "grad_norm": 4.311702547596846, + "learning_rate": 8.333333333333334e-06, + "loss": 0.7524, + "step": 10 + }, + { + "epoch": 1.1518987341772151, + "grad_norm": 3.822432777947372, + "learning_rate": 9.166666666666666e-06, + "loss": 0.6639, + "step": 11 + }, + { + "epoch": 1.2531645569620253, + "grad_norm": 3.5468998915875813, + "learning_rate": 1e-05, + "loss": 0.6936, + "step": 12 + }, + { + "epoch": 1.3544303797468356, + "grad_norm": 2.800787770940674, + "learning_rate": 9.997762161417517e-06, + "loss": 0.6682, + "step": 13 + }, + { + "epoch": 1.4556962025316456, + "grad_norm": 2.1669190884101175, + "learning_rate": 9.991050648838676e-06, + "loss": 0.5577, + "step": 14 + }, + { + "epoch": 1.5569620253164556, + "grad_norm": 2.087992110618876, + "learning_rate": 9.979871469976197e-06, + "loss": 0.5829, + "step": 15 + }, + { + "epoch": 1.6582278481012658, + "grad_norm": 2.7059991544282798, + "learning_rate": 9.964234631709188e-06, + "loss": 0.6041, + "step": 16 + }, + { + "epoch": 1.759493670886076, + "grad_norm": 2.2940983342523125, + "learning_rate": 9.944154131125643e-06, + "loss": 0.6291, + "step": 17 + }, + { + "epoch": 1.8607594936708862, + "grad_norm": 1.6808408200948324, + "learning_rate": 9.91964794299315e-06, + "loss": 0.6081, + "step": 18 + }, + { + "epoch": 1.9620253164556962, + "grad_norm": 1.6600609215129083, + "learning_rate": 9.890738003669029e-06, + "loss": 0.5927, + "step": 19 + }, + { + "epoch": 2.1012658227848102, + "grad_norm": 1.472819671631874, + "learning_rate": 9.857450191464337e-06, + "loss": 0.5682, + "step": 20 + }, + { + "epoch": 2.2025316455696204, + "grad_norm": 1.6799800601492856, + "learning_rate": 9.819814303479268e-06, + "loss": 0.5403, + "step": 21 + }, + { + "epoch": 2.3037974683544302, + "grad_norm": 1.3804440688611062, + "learning_rate": 9.777864028930705e-06, + "loss": 0.4855, + "step": 22 + }, + { + "epoch": 2.4050632911392404, + "grad_norm": 1.1432325373879264, + "learning_rate": 9.731636918995821e-06, + "loss": 0.5031, + "step": 23 + }, + { + "epoch": 2.5063291139240507, + "grad_norm": 0.9092916002520707, + "learning_rate": 9.681174353198687e-06, + "loss": 0.4571, + "step": 24 + }, + { + "epoch": 2.607594936708861, + "grad_norm": 1.1253148386623655, + "learning_rate": 9.626521502369984e-06, + "loss": 0.4672, + "step": 25 + }, + { + "epoch": 2.708860759493671, + "grad_norm": 1.4032465429020564, + "learning_rate": 9.567727288213005e-06, + "loss": 0.4384, + "step": 26 + }, + { + "epoch": 2.810126582278481, + "grad_norm": 1.1988699935589895, + "learning_rate": 9.504844339512096e-06, + "loss": 0.3985, + "step": 27 + }, + { + "epoch": 2.911392405063291, + "grad_norm": 0.8951456999785163, + "learning_rate": 9.437928945022772e-06, + "loss": 0.4792, + "step": 28 + }, + { + "epoch": 3.050632911392405, + "grad_norm": 0.9624502825360387, + "learning_rate": 9.36704100308565e-06, + "loss": 0.362, + "step": 29 + }, + { + "epoch": 3.151898734177215, + "grad_norm": 1.0438288012802868, + "learning_rate": 9.292243968009332e-06, + "loss": 0.3723, + "step": 30 + }, + { + "epoch": 3.2531645569620253, + "grad_norm": 0.9703176540776492, + "learning_rate": 9.213604793270196e-06, + "loss": 0.3369, + "step": 31 + }, + { + "epoch": 3.3544303797468356, + "grad_norm": 0.893163662422422, + "learning_rate": 9.131193871579975e-06, + "loss": 0.3604, + "step": 32 + }, + { + "epoch": 3.4556962025316453, + "grad_norm": 1.2422079407990139, + "learning_rate": 9.045084971874738e-06, + "loss": 0.3631, + "step": 33 + }, + { + "epoch": 3.5569620253164556, + "grad_norm": 0.9151822546049537, + "learning_rate": 8.955355173281709e-06, + "loss": 0.3553, + "step": 34 + }, + { + "epoch": 3.6582278481012658, + "grad_norm": 1.0114454402313395, + "learning_rate": 8.862084796122998e-06, + "loss": 0.4085, + "step": 35 + }, + { + "epoch": 3.759493670886076, + "grad_norm": 0.9443337928269486, + "learning_rate": 8.765357330018056e-06, + "loss": 0.2889, + "step": 36 + }, + { + "epoch": 3.8607594936708862, + "grad_norm": 0.865644384923322, + "learning_rate": 8.665259359149132e-06, + "loss": 0.2859, + "step": 37 + }, + { + "epoch": 3.962025316455696, + "grad_norm": 0.8599590553282734, + "learning_rate": 8.561880484756726e-06, + "loss": 0.36, + "step": 38 + }, + { + "epoch": 4.10126582278481, + "grad_norm": 0.8238766304670959, + "learning_rate": 8.455313244934324e-06, + "loss": 0.284, + "step": 39 + }, + { + "epoch": 4.2025316455696204, + "grad_norm": 0.8904225393629464, + "learning_rate": 8.345653031794292e-06, + "loss": 0.2657, + "step": 40 + }, + { + "epoch": 4.30379746835443, + "grad_norm": 0.8281971083836545, + "learning_rate": 8.232998006078998e-06, + "loss": 0.2237, + "step": 41 + }, + { + "epoch": 4.405063291139241, + "grad_norm": 0.9318352078173276, + "learning_rate": 8.117449009293668e-06, + "loss": 0.2345, + "step": 42 + }, + { + "epoch": 4.506329113924051, + "grad_norm": 0.9593192555567228, + "learning_rate": 7.99910947343957e-06, + "loss": 0.2498, + "step": 43 + }, + { + "epoch": 4.6075949367088604, + "grad_norm": 0.8527439601562876, + "learning_rate": 7.87808532842837e-06, + "loss": 0.2405, + "step": 44 + }, + { + "epoch": 4.708860759493671, + "grad_norm": 0.9473177017919994, + "learning_rate": 7.754484907260513e-06, + "loss": 0.2913, + "step": 45 + }, + { + "epoch": 4.810126582278481, + "grad_norm": 0.8102313751416961, + "learning_rate": 7.628418849052523e-06, + "loss": 0.286, + "step": 46 + }, + { + "epoch": 4.911392405063291, + "grad_norm": 0.820114624158469, + "learning_rate": 7.500000000000001e-06, + "loss": 0.2364, + "step": 47 + }, + { + "epoch": 5.050632911392405, + "grad_norm": 0.7927604968113404, + "learning_rate": 7.369343312364994e-06, + "loss": 0.2076, + "step": 48 + }, + { + "epoch": 5.151898734177215, + "grad_norm": 0.6375951563290391, + "learning_rate": 7.236565741578163e-06, + "loss": 0.1667, + "step": 49 + }, + { + "epoch": 5.253164556962025, + "grad_norm": 0.6651567314411381, + "learning_rate": 7.101786141547829e-06, + "loss": 0.1684, + "step": 50 + }, + { + "epoch": 5.3544303797468356, + "grad_norm": 0.7043980605084836, + "learning_rate": 6.965125158269619e-06, + "loss": 0.2039, + "step": 51 + }, + { + "epoch": 5.455696202531645, + "grad_norm": 0.7406146243351966, + "learning_rate": 6.8267051218319766e-06, + "loss": 0.1794, + "step": 52 + }, + { + "epoch": 5.556962025316456, + "grad_norm": 0.7560497079368976, + "learning_rate": 6.686649936914151e-06, + "loss": 0.1529, + "step": 53 + }, + { + "epoch": 5.658227848101266, + "grad_norm": 0.6135830262872792, + "learning_rate": 6.545084971874738e-06, + "loss": 0.1956, + "step": 54 + }, + { + "epoch": 5.759493670886076, + "grad_norm": 0.7112397939983846, + "learning_rate": 6.402136946530014e-06, + "loss": 0.1958, + "step": 55 + }, + { + "epoch": 5.860759493670886, + "grad_norm": 0.7940516397429221, + "learning_rate": 6.257933818722544e-06, + "loss": 0.217, + "step": 56 + }, + { + "epoch": 5.962025316455696, + "grad_norm": 0.7871752636384775, + "learning_rate": 6.112604669781572e-06, + "loss": 0.1946, + "step": 57 + }, + { + "epoch": 6.10126582278481, + "grad_norm": 0.5470125055488606, + "learning_rate": 5.9662795889777666e-06, + "loss": 0.1701, + "step": 58 + }, + { + "epoch": 6.2025316455696204, + "grad_norm": 0.5813020536626888, + "learning_rate": 5.819089557075689e-06, + "loss": 0.1426, + "step": 59 + }, + { + "epoch": 6.30379746835443, + "grad_norm": 0.5710246684545575, + "learning_rate": 5.671166329088278e-06, + "loss": 0.1184, + "step": 60 + }, + { + "epoch": 6.405063291139241, + "grad_norm": 0.8565249872894666, + "learning_rate": 5.522642316338268e-06, + "loss": 0.1978, + "step": 61 + }, + { + "epoch": 6.506329113924051, + "grad_norm": 0.7450325352987529, + "learning_rate": 5.373650467932122e-06, + "loss": 0.1521, + "step": 62 + }, + { + "epoch": 6.6075949367088604, + "grad_norm": 0.7228163461638277, + "learning_rate": 5.224324151752575e-06, + "loss": 0.1556, + "step": 63 + }, + { + "epoch": 6.708860759493671, + "grad_norm": 0.5931907548279679, + "learning_rate": 5.074797035076319e-06, + "loss": 0.1134, + "step": 64 + }, + { + "epoch": 6.810126582278481, + "grad_norm": 0.6442106835491783, + "learning_rate": 4.9252029649236835e-06, + "loss": 0.1085, + "step": 65 + }, + { + "epoch": 6.911392405063291, + "grad_norm": 0.6660065713778855, + "learning_rate": 4.775675848247427e-06, + "loss": 0.1296, + "step": 66 + }, + { + "epoch": 7.050632911392405, + "grad_norm": 0.513068792938708, + "learning_rate": 4.626349532067879e-06, + "loss": 0.1223, + "step": 67 + }, + { + "epoch": 7.151898734177215, + "grad_norm": 0.4707168636389631, + "learning_rate": 4.477357683661734e-06, + "loss": 0.0976, + "step": 68 + }, + { + "epoch": 7.253164556962025, + "grad_norm": 0.5584208200417776, + "learning_rate": 4.3288336709117246e-06, + "loss": 0.1576, + "step": 69 + }, + { + "epoch": 7.3544303797468356, + "grad_norm": 0.4948162292093854, + "learning_rate": 4.180910442924312e-06, + "loss": 0.1002, + "step": 70 + }, + { + "epoch": 7.455696202531645, + "grad_norm": 0.6003769756361554, + "learning_rate": 4.033720411022235e-06, + "loss": 0.0634, + "step": 71 + }, + { + "epoch": 7.556962025316456, + "grad_norm": 0.632817072117396, + "learning_rate": 3.887395330218429e-06, + "loss": 0.1324, + "step": 72 + }, + { + "epoch": 7.658227848101266, + "grad_norm": 0.5512268097086501, + "learning_rate": 3.7420661812774577e-06, + "loss": 0.0783, + "step": 73 + }, + { + "epoch": 7.759493670886076, + "grad_norm": 0.5935696380747312, + "learning_rate": 3.5978630534699873e-06, + "loss": 0.1258, + "step": 74 + }, + { + "epoch": 7.860759493670886, + "grad_norm": 0.43674489606470374, + "learning_rate": 3.4549150281252635e-06, + "loss": 0.1357, + "step": 75 + }, + { + "epoch": 7.962025316455696, + "grad_norm": 0.5040701068675334, + "learning_rate": 3.3133500630858507e-06, + "loss": 0.1221, + "step": 76 + }, + { + "epoch": 8.10126582278481, + "grad_norm": 0.5537439646466557, + "learning_rate": 3.173294878168025e-06, + "loss": 0.0895, + "step": 77 + }, + { + "epoch": 8.20253164556962, + "grad_norm": 0.3734160602926547, + "learning_rate": 3.0348748417303826e-06, + "loss": 0.1119, + "step": 78 + }, + { + "epoch": 8.30379746835443, + "grad_norm": 0.36353809777994334, + "learning_rate": 2.8982138584521734e-06, + "loss": 0.0546, + "step": 79 + }, + { + "epoch": 8.405063291139241, + "grad_norm": 0.5432026494165365, + "learning_rate": 2.7634342584218364e-06, + "loss": 0.0701, + "step": 80 + }, + { + "epoch": 8.50632911392405, + "grad_norm": 0.7187854238009306, + "learning_rate": 2.6306566876350072e-06, + "loss": 0.088, + "step": 81 + }, + { + "epoch": 8.60759493670886, + "grad_norm": 0.6028065371137555, + "learning_rate": 2.5000000000000015e-06, + "loss": 0.0989, + "step": 82 + }, + { + "epoch": 8.708860759493671, + "grad_norm": 0.6029218748519589, + "learning_rate": 2.371581150947476e-06, + "loss": 0.1138, + "step": 83 + }, + { + "epoch": 8.810126582278482, + "grad_norm": 0.44030990822373384, + "learning_rate": 2.245515092739488e-06, + "loss": 0.0778, + "step": 84 + }, + { + "epoch": 8.91139240506329, + "grad_norm": 0.44206891285952626, + "learning_rate": 2.1219146715716332e-06, + "loss": 0.1197, + "step": 85 + }, + { + "epoch": 9.050632911392405, + "grad_norm": 0.4181481470988038, + "learning_rate": 2.0008905265604316e-06, + "loss": 0.0645, + "step": 86 + }, + { + "epoch": 9.151898734177216, + "grad_norm": 0.47866066389040807, + "learning_rate": 1.8825509907063328e-06, + "loss": 0.0707, + "step": 87 + }, + { + "epoch": 9.253164556962025, + "grad_norm": 0.4855471053569369, + "learning_rate": 1.7670019939210025e-06, + "loss": 0.0592, + "step": 88 + }, + { + "epoch": 9.354430379746836, + "grad_norm": 0.42763865169197446, + "learning_rate": 1.6543469682057105e-06, + "loss": 0.107, + "step": 89 + }, + { + "epoch": 9.455696202531646, + "grad_norm": 0.3690131276831282, + "learning_rate": 1.544686755065677e-06, + "loss": 0.0515, + "step": 90 + }, + { + "epoch": 9.556962025316455, + "grad_norm": 0.3883137017573927, + "learning_rate": 1.438119515243277e-06, + "loss": 0.1044, + "step": 91 + }, + { + "epoch": 9.658227848101266, + "grad_norm": 0.7347299174387595, + "learning_rate": 1.3347406408508695e-06, + "loss": 0.0741, + "step": 92 + }, + { + "epoch": 9.759493670886076, + "grad_norm": 0.4812162548965024, + "learning_rate": 1.234642669981946e-06, + "loss": 0.1096, + "step": 93 + }, + { + "epoch": 9.860759493670885, + "grad_norm": 0.369367326407901, + "learning_rate": 1.137915203877003e-06, + "loss": 0.0738, + "step": 94 + }, + { + "epoch": 9.962025316455696, + "grad_norm": 0.48681431404942604, + "learning_rate": 1.044644826718295e-06, + "loss": 0.0798, + "step": 95 + }, + { + "epoch": 10.10126582278481, + "grad_norm": 0.38416364988319346, + "learning_rate": 9.549150281252633e-07, + "loss": 0.0772, + "step": 96 + }, + { + "epoch": 10.20253164556962, + "grad_norm": 0.2972730096642589, + "learning_rate": 8.688061284200266e-07, + "loss": 0.0706, + "step": 97 + }, + { + "epoch": 10.30379746835443, + "grad_norm": 0.3687593689489182, + "learning_rate": 7.863952067298042e-07, + "loss": 0.0854, + "step": 98 + }, + { + "epoch": 10.405063291139241, + "grad_norm": 0.28383029271802473, + "learning_rate": 7.077560319906696e-07, + "loss": 0.0463, + "step": 99 + }, + { + "epoch": 10.50632911392405, + "grad_norm": 0.329282008425657, + "learning_rate": 6.329589969143518e-07, + "loss": 0.0642, + "step": 100 + }, + { + "epoch": 10.60759493670886, + "grad_norm": 0.2500705047947425, + "learning_rate": 5.620710549772295e-07, + "loss": 0.0462, + "step": 101 + }, + { + "epoch": 10.708860759493671, + "grad_norm": 0.3989612983696247, + "learning_rate": 4.951556604879049e-07, + "loss": 0.1179, + "step": 102 + }, + { + "epoch": 10.810126582278482, + "grad_norm": 0.2662828923303931, + "learning_rate": 4.322727117869951e-07, + "loss": 0.0675, + "step": 103 + }, + { + "epoch": 10.91139240506329, + "grad_norm": 0.2711704696179231, + "learning_rate": 3.734784976300165e-07, + "loss": 0.0693, + "step": 104 + }, + { + "epoch": 11.050632911392405, + "grad_norm": 0.34511642958777133, + "learning_rate": 3.18825646801314e-07, + "loss": 0.0957, + "step": 105 + }, + { + "epoch": 11.151898734177216, + "grad_norm": 0.2889242607992141, + "learning_rate": 2.6836308100417874e-07, + "loss": 0.041, + "step": 106 + }, + { + "epoch": 11.253164556962025, + "grad_norm": 0.24450548412580397, + "learning_rate": 2.2213597106929608e-07, + "loss": 0.0564, + "step": 107 + }, + { + "epoch": 11.354430379746836, + "grad_norm": 0.35934542044635454, + "learning_rate": 1.801856965207338e-07, + "loss": 0.0804, + "step": 108 + }, + { + "epoch": 11.455696202531646, + "grad_norm": 0.26298606930327606, + "learning_rate": 1.4254980853566248e-07, + "loss": 0.041, + "step": 109 + }, + { + "epoch": 11.556962025316455, + "grad_norm": 0.31777226755411614, + "learning_rate": 1.0926199633097156e-07, + "loss": 0.0903, + "step": 110 + }, + { + "epoch": 11.658227848101266, + "grad_norm": 0.3193608086114228, + "learning_rate": 8.035205700685167e-08, + "loss": 0.0721, + "step": 111 + }, + { + "epoch": 11.759493670886076, + "grad_norm": 0.25486180463366104, + "learning_rate": 5.584586887435739e-08, + "loss": 0.0657, + "step": 112 + }, + { + "epoch": 11.860759493670885, + "grad_norm": 0.25091083569651457, + "learning_rate": 3.576536829081323e-08, + "loss": 0.078, + "step": 113 + }, + { + "epoch": 11.962025316455696, + "grad_norm": 0.25614891878084256, + "learning_rate": 2.012853002380466e-08, + "loss": 0.0652, + "step": 114 + }, + { + "epoch": 12.10126582278481, + "grad_norm": 0.3196686889199045, + "learning_rate": 8.949351161324227e-09, + "loss": 0.0842, + "step": 115 + }, + { + "epoch": 12.20253164556962, + "grad_norm": 0.306975262273726, + "learning_rate": 2.237838582483387e-09, + "loss": 0.0994, + "step": 116 + }, + { + "epoch": 12.30379746835443, + "grad_norm": 0.24537391909796794, + "learning_rate": 0.0, + "loss": 0.0314, + "step": 117 + }, + { + "epoch": 12.30379746835443, + "step": 117, + "total_flos": 4.902218749915955e+16, + "train_loss": 0.26980868625080484, + "train_runtime": 2812.7387, + "train_samples_per_second": 1.46, + "train_steps_per_second": 0.042 + } + ], + "logging_steps": 1, + "max_steps": 117, + "num_input_tokens_seen": 0, + "num_train_epochs": 13, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 4.902218749915955e+16, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..956b850 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e359e3476ac1fbe0beec77a258cb644e16d901568de9a8025204ac5c6c8742c +size 7288 diff --git a/training_loss.png b/training_loss.png new file mode 100644 index 0000000..8a3034b Binary files /dev/null and b/training_loss.png differ diff --git a/vocab.json b/vocab.json new file mode 100644 index 0000000..6c49fc6 --- /dev/null +++ b/vocab.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910 +size 2776833