commit d042b72da44542be9102cd627ceda72cb6c72a8b Author: ModelHub XC Date: Tue Jul 7 04:14:14 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: JunHowie/Qwen3-14B-Instruct-2512-SFT Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..31b0f78 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,58 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text + + +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text + +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +tokenizer.json filter=lfs diff=lfs merge=lfs -text +model-00001-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text +merges.txt filter=lfs diff=lfs merge=lfs -text +model-00002-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text +vocab.json filter=lfs diff=lfs merge=lfs -text +model-00004-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text +model-00006-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text +training_args.bin filter=lfs diff=lfs merge=lfs -text +model-00003-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text +model-00005-of-00006.safetensors filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..c81af14 --- /dev/null +++ b/README.md @@ -0,0 +1,117 @@ +--- +library_name: transformers +license: apache-2.0 +license_link: https://huggingface.co/Qwen/Qwen3-8B/blob/main/LICENSE +pipeline_tag: text-generation +base_model: +- Qwen/Qwen3-8B +--- +# Qwen3-14B-Instruct-2512-SFT + +**NOTE：This model is the Instruct-aligned variant, and it will not generate ```` blocks in its outputs. +Additionally, there is no need to specify enable_thinking=False anymore.** + + + +Among them, the 8B and 14B SFT and DFT variants are obtained via full-parameter fine-tuning, while the 32B models are trained using LoRA due to hardware resource constraints.
+The dataset used is the Chinese Distillation Dataset based on Qwen3-235B-2507.available at:[**Chinese-Qwen3-235B-2507-Distill-data-110k**](https://www.modelscope.cn/datasets/swift/Chinese-Qwen3-235B-2507-Distill-data-110k) +
+For details and code regarding model training and quantization, please see[Training and Quantization Guide](https://www.modelscope.cn/learn/3000) +
+Here is the list of models released in this version:
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model4-bit AWQ8-bit FP8GPTQNVIDIA FP4Weight-Activation
AWQAWQ-asymINT4INT8NVFP4NVFP4-A16W4A16W8A8
Qwen3-8B-Instruct-2512-DFTAWQawq-asymFP8GPTQ(int4)GPTQ(int8)NVFP4NVFP4A16W4A16W8A8
Qwen3-8B-Instruct-2512-SFTAWQawq-asymFP8GPTQ(int4)GPTQ(int8)NVFP4NVFP4A16W4A16W8A8
Qwen3-14B-Instruct-2512-DFTAWQawq-asymFP8GPTQ(int4)GPTQ(int8)NVFP4NVFP4A16W4A16W8A8
Qwen3-14B-Instruct-2512-SFTAWQawq-asymFP8GPTQ(int4)GPTQ(int8)NVFP4NVFP4A16W4A16W8A8
Qwen3-32B-Instruct-2512-DFTAWQawq-asymFP8GPTQ(int4)GPTQ(int8)NVFP4NVFP4A16W4A16W8A8
+ + + +### 【Dependencies】 +``` +vllm>=0.10.2 +transformers>=4.56.1 +``` + + + diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..b54f913 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/args.json b/args.json new file mode 100644 index 0000000..a0b6f75 --- /dev/null +++ b/args.json @@ -0,0 +1,390 @@ +{ + "output_dir": "/root/data/output/14B-SFT-Zero3/v1-20251202-132624", + "overwrite_output_dir": false, + "do_train": false, + "do_eval": false, + "do_predict": false, + "eval_strategy": "steps", + "prediction_loss_only": false, + "per_device_train_batch_size": 4, + "per_device_eval_batch_size": 1, + "per_gpu_train_batch_size": null, + "per_gpu_eval_batch_size": null, + "gradient_accumulation_steps": 1, + "eval_accumulation_steps": null, + "eval_delay": 0, + "torch_empty_cache_steps": null, + "learning_rate": 1e-05, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "adam_beta2": 0.95, + "adam_epsilon": 1e-08, + "max_grad_norm": 1.0, + "num_train_epochs": 1.0, + "max_steps": -1, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": null, + "warmup_ratio": 0.0, + "warmup_steps": 250, + "log_level": "passive", + "log_level_replica": "warning", + "log_on_each_node": true, + "logging_dir": "/root/data/output/14B-SFT-Zero3/v1-20251202-132624/runs", + "logging_strategy": "steps", + "logging_first_step": true, + "logging_steps": 10, + "logging_nan_inf_filter": true, + "save_strategy": "steps", + "save_steps": 200.0, + "save_total_limit": 5, + "save_safetensors": true, + "save_on_each_node": false, + "save_only_model": true, + "restore_callback_states_from_checkpoint": false, + "no_cuda": false, + "use_cpu": false, + "use_mps_device": false, + "seed": 42, + "data_seed": 42, + "jit_mode_eval": false, + "bf16": true, + "fp16": false, + "fp16_opt_level": "O1", + "half_precision_backend": "auto", + "bf16_full_eval": false, + "fp16_full_eval": false, + "tf32": null, + "local_rank": 0, + "ddp_backend": null, + "tpu_num_cores": null, + "tpu_metrics_debug": false, + "debug": null, + "dataloader_drop_last": false, + "eval_steps": 200.0, + "dataloader_num_workers": 4, + "dataloader_prefetch_factor": null, + "past_index": -1, + "run_name": "/root/data/output/14B-SFT-Zero3/v1-20251202-132624", + "disable_tqdm": null, + "remove_unused_columns": true, + "label_names": null, + "load_best_model_at_end": false, + "metric_for_best_model": "loss", + "greater_is_better": false, + "ignore_data_skip": false, + "fsdp": null, + "fsdp_min_num_params": 0, + "fsdp_config": null, + "fsdp_transformer_layer_cls_to_wrap": null, + "accelerator_config": { + "dispatch_batches": false + }, + "parallelism_config": null, + "deepspeed": { + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 1000, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1 + }, + "bf16": { + "enabled": "auto" + }, + "zero_optimization": { + "stage": 3, + "offload_optimizer": { + "device": "none", + "pin_memory": true + }, + "offload_param": { + "device": "none", + "pin_memory": true + }, + "overlap_comm": false, + "contiguous_gradients": true, + "sub_group_size": 1000000000.0, + "reduce_bucket_size": "auto", + "zero_quantized_weights": false, + "zero_quantized_gradients": false, + "stage3_prefetch_bucket_size": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_max_live_parameters": 1000000000.0, + "stage3_max_reuse_distance": 1000000000.0, + "stage3_gather_16bit_weights_on_model_save": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false + }, + "label_smoothing_factor": 0.0, + "optim": "adamw_torch_fused", + "optim_args": null, + "adafactor": false, + "group_by_length": true, + "length_column_name": "length", + "report_to": [ + "tensorboard" + ], + "project": "huggingface", + "trackio_space_id": "trackio", + "ddp_find_unused_parameters": false, + "ddp_bucket_cap_mb": null, + "ddp_broadcast_buffers": null, + "dataloader_pin_memory": true, + "dataloader_persistent_workers": false, + "skip_memory_metrics": true, + "use_legacy_prediction_loop": false, + "push_to_hub": false, + "resume_from_checkpoint": null, + "hub_model_id": null, + "hub_strategy": "every_save", + "hub_token": null, + "hub_private_repo": null, + "hub_always_push": false, + "hub_revision": null, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": null, + "include_inputs_for_metrics": false, + "include_for_metrics": [], + "eval_do_concat_batches": true, + "fp16_backend": "auto", + "push_to_hub_model_id": null, + "push_to_hub_organization": null, + "push_to_hub_token": null, + "mp_parameters": "", + "auto_find_batch_size": false, + "full_determinism": false, + "torchdynamo": null, + "ray_scope": "last", + "ddp_timeout": 18000000, + "torch_compile": false, + "torch_compile_backend": null, + "torch_compile_mode": null, + "include_tokens_per_second": false, + "include_num_input_tokens_seen": false, + "neftune_noise_alpha": null, + "optim_target_modules": null, + "batch_eval_metrics": false, + "eval_on_start": false, + "use_liger_kernel": true, + "liger_kernel_config": null, + "eval_use_gather_object": false, + "average_tokens_across_devices": true, + "sortish_sampler": false, + "predict_with_generate": false, + "generation_max_length": null, + "generation_num_beams": null, + "generation_config": null, + "tuner_backend": "peft", + "vit_gradient_checkpointing": null, + "router_aux_loss_coef": 0.0, + "enable_dft_loss": false, + "enable_channel_loss": false, + "check_model": true, + "acc_strategy": "token", + "train_dataloader_shuffle": true, + "max_epochs": null, + "aligner_lr": null, + "vit_lr": null, + "use_logits_to_keep": null, + "ds3_gather_for_generation": true, + "resume_only_model": false, + "optimizer": null, + "loss_type": null, + "metric": null, + "eval_use_evalscope": false, + "eval_dataset": [], + "eval_dataset_args": null, + "eval_limit": null, + "eval_generation_config": null, + "extra_eval_args": null, + "use_flash_ckpt": false, + "use_ray": false, + "ray_exp_name": null, + "device_groups": null, + "model": "/share/new_models/qwen3/Qwen3-14B", + "model_type": "qwen3_nothinking", + "model_revision": null, + "task_type": "causal_lm", + "torch_dtype": "bfloat16", + "attn_impl": "flash_attn", + "new_special_tokens": [], + "num_labels": null, + "problem_type": null, + "rope_scaling": null, + "device_map": null, + "max_memory": {}, + "max_model_len": null, + "local_repo_path": null, + "init_strategy": null, + "template": "qwen3_nothinking", + "system": null, + "max_length": 1536, + "truncation_strategy": "delete", + "max_pixels": null, + "agent_template": null, + "norm_bbox": null, + "use_chat_template": true, + "padding_free": false, + "padding_side": "right", + "loss_scale": "default", + "sequence_parallel_size": 1, + "response_prefix": null, + "template_backend": "swift", + "dataset": [ + "/root/data/datasets/qwen3_235b_2507_distill_110k.jsonl" + ], + "val_dataset": [], + "split_dataset_ratio": 0.02, + "dataset_num_proc": 1, + "load_from_cache_file": true, + "dataset_shuffle": true, + "val_dataset_shuffle": false, + "streaming": false, + "interleave_prob": null, + "stopping_strategy": "first_exhausted", + "shuffle_buffer_size": 1000, + "download_mode": "reuse_dataset_if_exists", + "columns": {}, + "strict": false, + "model_name": [ + "Qwen3-14B-SFT" + ], + "model_author": [ + "JunHowie" + ], + "custom_dataset_info": [], + "quant_method": null, + "quant_bits": null, + "hqq_axis": null, + "bnb_4bit_compute_dtype": "bfloat16", + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_use_double_quant": true, + "bnb_4bit_quant_storage": null, + "max_new_tokens": 64, + "temperature": 0.0, + "top_k": null, + "top_p": null, + "repetition_penalty": null, + "num_beams": 1, + "stream": false, + "stop_words": [], + "logprobs": false, + "top_logprobs": null, + "ckpt_dir": null, + "lora_modules": [], + "train_type": "full", + "adapters": [], + "external_plugins": [], + "model_kwargs": {}, + "load_args": false, + "load_data_args": false, + "packing": false, + "packing_length": null, + "lazy_tokenize": false, + "cached_dataset": [], + "custom_register_path": [], + "use_hf": false, + "ignore_args_error": false, + "use_swift_lora": false, + "freeze_parameters": [], + "freeze_parameters_regex": null, + "freeze_parameters_ratio": 0.0, + "trainable_parameters": [], + "trainable_parameters_regex": null, + "freeze_llm": false, + "freeze_vit": true, + "freeze_aligner": true, + "target_modules": [ + "all-linear" + ], + "target_regex": null, + "target_parameters": null, + "modules_to_save": [], + "lora_rank": 8, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_bias": "none", + "lora_dtype": null, + "lorap_lr_ratio": null, + "use_rslora": false, + "use_dora": false, + "lora_ga_batch_size": 2, + "lora_ga_iters": 2, + "lora_ga_max_length": 1024, + "lora_ga_direction": "ArB2r", + "lora_ga_scale": "stable", + "lora_ga_stable_gamma": 16, + "init_weights": true, + "fourier_n_frequency": 2000, + "fourier_scaling": 300.0, + "boft_block_size": 4, + "boft_block_num": 0, + "boft_n_butterfly_factor": 1, + "boft_dropout": 0.0, + "vera_rank": 256, + "vera_projection_prng_key": 0, + "vera_dropout": 0.0, + "vera_d_initial": 0.1, + "adapter_act": "gelu", + "adapter_length": 128, + "use_galore": false, + "galore_target_modules": null, + "galore_rank": 128, + "galore_update_proj_gap": 50, + "galore_scale": 1.0, + "galore_proj_type": "std", + "galore_optim_per_parameter": false, + "galore_with_embedding": false, + "galore_quantization": false, + "galore_proj_quant": false, + "galore_proj_bits": 4, + "galore_proj_group_size": 256, + "galore_cos_threshold": 0.4, + "galore_gamma_proj": 2, + "galore_queue_size": 5, + "adalora_target_r": 8, + "adalora_init_r": 12, + "adalora_tinit": 0, + "adalora_tfinal": 0, + "adalora_deltaT": 1, + "adalora_beta1": 0.85, + "adalora_beta2": 0.85, + "adalora_orth_reg_weight": 0.5, + "llamapro_num_new_blocks": 4, + "llamapro_num_groups": null, + "lisa_activated_layers": 0, + "lisa_step_interval": 20, + "reft_layer_key": null, + "reft_layers": null, + "reft_rank": 4, + "reft_intervention_type": "LoreftIntervention", + "reft_args": null, + "swanlab_token": null, + "swanlab_project": null, + "swanlab_workspace": null, + "swanlab_exp_name": null, + "swanlab_lark_webhook_url": null, + "swanlab_lark_secret": null, + "swanlab_mode": "cloud", + "add_version": true, + "create_checkpoint_symlink": false, + "zero_hpz_partition_size": null, + "deepspeed_autotp_size": null, + "early_stop_interval": null, + "rank": 0, + "global_world_size": 4, + "local_world_size": 4, + "model_suffix": "Qwen3-14B", + "model_info": "ModelInfo(model_type='qwen3_nothinking', model_dir='/share/new_models/qwen3/Qwen3-14B', torch_dtype=torch.bfloat16, max_model_len=40960, quant_method=None, quant_bits=None, rope_scaling=None, is_moe_model=False, config=None, task_type='causal_lm', num_labels=None)", + "model_meta": "ModelMeta(model_type='qwen3_nothinking', model_groups=[ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-30B-A3B-Instruct-2507', hf_model_id='Qwen/Qwen3-30B-A3B-Instruct-2507', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-30B-A3B-Instruct-2507-FP8', hf_model_id='Qwen/Qwen3-30B-A3B-Instruct-2507-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-235B-A22B-Instruct-2507', hf_model_id='Qwen/Qwen3-235B-A22B-Instruct-2507', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-235B-A22B-Instruct-2507-FP8', hf_model_id='Qwen/Qwen3-235B-A22B-Instruct-2507-FP8', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='swift/Qwen3-235B-A22B-Instruct-2507-AWQ', hf_model_id=None, model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[]), ModelGroup(models=[Model(ms_model_id='Qwen/Qwen3-4B-Instruct-2507', hf_model_id='Qwen/Qwen3-4B-Instruct-2507', model_path=None, ms_revision=None, hf_revision=None), Model(ms_model_id='Qwen/Qwen3-4B-Instruct-2507-FP8', hf_model_id='Qwen/Qwen3-4B-Instruct-2507-FP8', model_path=None, ms_revision=None, hf_revision=None)], ignore_patterns=None, requires=None, tags=[])], template='qwen3_nothinking', get_function=, model_arch=None, architectures=['Qwen3MoeForCausalLM', 'Qwen3ForCausalLM'], additional_saved_files=[], torch_dtype=None, is_multimodal=False, is_reward=False, is_reranker=False, task_type=None, ignore_patterns=None, requires=['transformers>=4.51'], tags=[])", + "model_dir": "/share/new_models/qwen3/Qwen3-14B", + "hub": "", + "evaluation_strategy": "steps", + "training_args": "Seq2SeqTrainingArguments(output_dir='/root/data/output/14B-SFT-Zero3/v1-20251202-132624', overwrite_output_dir=False, do_train=False, do_eval=True, do_predict=False, eval_strategy=, prediction_loss_only=False, per_device_train_batch_size=4, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=1, eval_accumulation_steps=None, eval_delay=0, torch_empty_cache_steps=None, learning_rate=1e-05, weight_decay=0.1, adam_beta1=0.9, adam_beta2=0.95, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=, lr_scheduler_kwargs=None, warmup_ratio=0.0, warmup_steps=250, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='/root/data/output/14B-SFT-Zero3/v1-20251202-132624/runs', logging_strategy=, logging_first_step=True, logging_steps=10, logging_nan_inf_filter=True, save_strategy=, save_steps=200, save_total_limit=5, save_safetensors=True, save_on_each_node=False, save_only_model=True, restore_callback_states_from_checkpoint=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=42, data_seed=42, jit_mode_eval=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=200, dataloader_num_workers=4, dataloader_prefetch_factor=10, past_index=-1, run_name='/root/data/output/14B-SFT-Zero3/v1-20251202-132624', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model='loss', greater_is_better=False, ignore_data_skip=False, fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, accelerator_config=AcceleratorConfig(split_batches=False, dispatch_batches=False, even_batches=True, use_seedable_sampler=True, non_blocking=False, gradient_accumulation_kwargs=None, use_configured_state=False), parallelism_config=None, deepspeed={'fp16': {'enabled': 'auto', 'loss_scale': 0, 'loss_scale_window': 1000, 'initial_scale_power': 16, 'hysteresis': 2, 'min_loss_scale': 1}, 'bf16': {'enabled': 'auto'}, 'zero_optimization': {'stage': 3, 'offload_optimizer': {'device': 'none', 'pin_memory': True}, 'offload_param': {'device': 'none', 'pin_memory': True}, 'overlap_comm': False, 'contiguous_gradients': True, 'sub_group_size': 1000000000.0, 'reduce_bucket_size': 'auto', 'zero_quantized_weights': False, 'zero_quantized_gradients': False, 'stage3_prefetch_bucket_size': 'auto', 'stage3_param_persistence_threshold': 'auto', 'stage3_max_live_parameters': 1000000000.0, 'stage3_max_reuse_distance': 1000000000.0, 'stage3_gather_16bit_weights_on_model_save': True}, 'gradient_accumulation_steps': 'auto', 'gradient_clipping': 'auto', 'steps_per_print': 2000, 'train_batch_size': 'auto', 'train_micro_batch_size_per_gpu': 'auto', 'wall_clock_breakdown': False}, label_smoothing_factor=0.0, optim=, optim_args=None, adafactor=False, group_by_length=True, length_column_name='length', report_to=['tensorboard'], project='huggingface', trackio_space_id='trackio', ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, dataloader_persistent_workers=False, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=, hub_token=None, hub_private_repo=None, hub_always_push=False, hub_revision=None, gradient_checkpointing=True, gradient_checkpointing_kwargs=None, include_inputs_for_metrics=False, include_for_metrics=[], eval_do_concat_batches=True, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=18000000, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, include_tokens_per_second=None, include_num_input_tokens_seen=None, neftune_noise_alpha=None, optim_target_modules=None, batch_eval_metrics=False, eval_on_start=False, use_liger_kernel=True, liger_kernel_config=None, eval_use_gather_object=False, average_tokens_across_devices=None, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, tuner_backend='peft', vit_gradient_checkpointing=True, router_aux_loss_coef=0.0, enable_dft_loss=False, enable_channel_loss=False, check_model=True, acc_strategy='token', train_dataloader_shuffle=True, max_epochs=None, aligner_lr=None, vit_lr=None, use_logits_to_keep=None, ds3_gather_for_generation=True, resume_only_model=False, optimizer=None, loss_type=None, metric=None, eval_use_evalscope=False, eval_dataset=[], eval_dataset_args=None, eval_limit=None, eval_generation_config=None, extra_eval_args=None, use_flash_ckpt=False, sft_alpha=0, chord_sft_dataset=[], chord_sft_per_device_train_batch_size=None, chord_enable_phi_function=False, chord_mu_warmup_steps=None, chord_mu_decay_steps=None, chord_mu_peak=None, chord_mu_valley=None, train_type='full', local_repo_path=None, galore_config=None, padding_side='right', padding_free=False, task_type='causal_lm', problem_type=None)" +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..699ff8d --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..4f4f3a9 --- /dev/null +++ b/config.json @@ -0,0 +1,72 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 17408, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 40, + "model_type": "qwen3", + "num_attention_heads": 40, + "num_hidden_layers": 40, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "4.57.1", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..9e289a1 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "4.57.1" +} diff --git a/merges.txt b/merges.txt new file mode 100644 index 0000000..80c1a19 --- /dev/null +++ b/merges.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5 +size 1671853 diff --git a/model-00001-of-00006.safetensors b/model-00001-of-00006.safetensors new file mode 100644 index 0000000..631c587 --- /dev/null +++ b/model-00001-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67637ba983e2229d59ad380bbec7910a3518c3c79713e1391fb4e64b5b403b72 +size 4984780784 diff --git a/model-00002-of-00006.safetensors b/model-00002-of-00006.safetensors new file mode 100644 index 0000000..60e2eff --- /dev/null +++ b/model-00002-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72f0c307b8dc51f9d9c6a3fc22671b5c1603759bb24e69393e7c26d20ff02a63 +size 4980892048 diff --git a/model-00003-of-00006.safetensors b/model-00003-of-00006.safetensors new file mode 100644 index 0000000..060e758 --- /dev/null +++ b/model-00003-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:591950b4a2fa9363a9ae83d60f37d12b7436581be60283c70d427b45399a3e68 +size 4928485104 diff --git a/model-00004-of-00006.safetensors b/model-00004-of-00006.safetensors new file mode 100644 index 0000000..43bab84 --- /dev/null +++ b/model-00004-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3300fd8ee6dbb85a04e3bc220ad3ee1b14aed38bbc1860428aab2fa87dba77d2 +size 4980892112 diff --git a/model-00005-of-00006.safetensors b/model-00005-of-00006.safetensors new file mode 100644 index 0000000..1a9207a --- /dev/null +++ b/model-00005-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3d14b5fa24a72b3d39b99903f35dd587785206da8d5a52cce8f933a2f3be790 +size 4928485104 diff --git a/model-00006-of-00006.safetensors b/model-00006-of-00006.safetensors new file mode 100644 index 0000000..2567d8d --- /dev/null +++ b/model-00006-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da9cf8c4b4b2586a9e4a8451b67939c07c5ffb8571d223eefc7db7efe8405af9 +size 4733130504 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..e9a468d --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,451 @@ +{ + "metadata": { + "total_parameters": 424960, + "total_size": 29536614400 + }, + "weight_map": { + "lm_head.weight": "model-00006-of-00006.safetensors", + "model.embed_tokens.weight": "model-00001-of-00006.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.20.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.21.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.27.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.28.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.30.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.30.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.30.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.31.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.32.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.33.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.34.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.35.input_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.35.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.35.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.36.input_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.36.self_attn.k_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.36.self_attn.q_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.input_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.37.self_attn.k_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.self_attn.q_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.input_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.38.self_attn.k_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.self_attn.q_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.input_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.39.self_attn.k_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.self_attn.q_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.norm.weight": "model-00006-of-00006.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..ac23c0a --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..cd71f61 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4 +size 11422654 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ddaf698 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,239 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..ef96fe6 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,4683 @@ +{ + "best_global_step": 6200, + "best_metric": 0.57034385, + "best_model_checkpoint": "/root/data/output/14B-SFT-Zero3/v1-20251202-132624/checkpoint-6200", + "epoch": 1.0, + "eval_steps": 200, + "global_step": 6228, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00016056518946692356, + "grad_norm": 4.759260503459119, + "learning_rate": 4e-08, + "loss": 0.864223837852478, + "step": 1 + }, + { + "epoch": 0.0016056518946692357, + "grad_norm": 6.708539221917098, + "learning_rate": 4.0000000000000003e-07, + "loss": 0.884010738796658, + "step": 10 + }, + { + "epoch": 0.0032113037893384713, + "grad_norm": 5.882288099451299, + "learning_rate": 8.000000000000001e-07, + "loss": 0.8287239074707031, + "step": 20 + }, + { + "epoch": 0.004816955684007707, + "grad_norm": 2.9474128729077966, + "learning_rate": 1.2000000000000002e-06, + "loss": 0.7924188613891602, + "step": 30 + }, + { + "epoch": 0.006422607578676943, + "grad_norm": 2.3311716309146617, + "learning_rate": 1.6000000000000001e-06, + "loss": 0.8175813674926757, + "step": 40 + }, + { + "epoch": 0.00802825947334618, + "grad_norm": 2.1516347676740146, + "learning_rate": 2.0000000000000003e-06, + "loss": 0.723994779586792, + "step": 50 + }, + { + "epoch": 0.009633911368015413, + "grad_norm": 2.5263601194756258, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.7877510070800782, + "step": 60 + }, + { + "epoch": 0.01123956326268465, + "grad_norm": 1.9776603721255415, + "learning_rate": 2.8000000000000003e-06, + "loss": 0.7068247318267822, + "step": 70 + }, + { + "epoch": 0.012845215157353885, + "grad_norm": 1.729578271108341, + "learning_rate": 3.2000000000000003e-06, + "loss": 0.7023281574249267, + "step": 80 + }, + { + "epoch": 0.014450867052023121, + "grad_norm": 2.1522946923251585, + "learning_rate": 3.6000000000000003e-06, + "loss": 0.7004528045654297, + "step": 90 + }, + { + "epoch": 0.01605651894669236, + "grad_norm": 2.0048746604259002, + "learning_rate": 4.000000000000001e-06, + "loss": 0.7160505771636962, + "step": 100 + }, + { + "epoch": 0.017662170841361593, + "grad_norm": 2.1276390555471667, + "learning_rate": 4.4e-06, + "loss": 0.6201703071594238, + "step": 110 + }, + { + "epoch": 0.019267822736030827, + "grad_norm": 2.3931953254109466, + "learning_rate": 4.800000000000001e-06, + "loss": 0.7100973129272461, + "step": 120 + }, + { + "epoch": 0.020873474630700065, + "grad_norm": 1.7632629841494898, + "learning_rate": 5.2e-06, + "loss": 0.6558451652526855, + "step": 130 + }, + { + "epoch": 0.0224791265253693, + "grad_norm": 2.009414501812128, + "learning_rate": 5.600000000000001e-06, + "loss": 0.7142935276031495, + "step": 140 + }, + { + "epoch": 0.024084778420038536, + "grad_norm": 2.160814354773918, + "learning_rate": 6e-06, + "loss": 0.6885526657104493, + "step": 150 + }, + { + "epoch": 0.02569043031470777, + "grad_norm": 1.8014277771410736, + "learning_rate": 6.4000000000000006e-06, + "loss": 0.7136541366577148, + "step": 160 + }, + { + "epoch": 0.027296082209377008, + "grad_norm": 2.018318003300108, + "learning_rate": 6.800000000000001e-06, + "loss": 0.6216143608093262, + "step": 170 + }, + { + "epoch": 0.028901734104046242, + "grad_norm": 1.952369538810581, + "learning_rate": 7.2000000000000005e-06, + "loss": 0.6702580928802491, + "step": 180 + }, + { + "epoch": 0.03050738599871548, + "grad_norm": 1.984247792321809, + "learning_rate": 7.600000000000001e-06, + "loss": 0.6995386123657227, + "step": 190 + }, + { + "epoch": 0.03211303789338472, + "grad_norm": 1.7794659354500173, + "learning_rate": 8.000000000000001e-06, + "loss": 0.6760697841644288, + "step": 200 + }, + { + "epoch": 0.03211303789338472, + "eval_loss": 0.6378431916236877, + "eval_runtime": 116.2437, + "eval_samples_per_second": 17.403, + "eval_steps_per_second": 4.353, + "eval_token_acc": 0.7867874882579367, + "step": 200 + }, + { + "epoch": 0.03371868978805395, + "grad_norm": 2.031179863122134, + "learning_rate": 8.400000000000001e-06, + "loss": 0.6848911285400391, + "step": 210 + }, + { + "epoch": 0.035324341682723186, + "grad_norm": 1.769054282614552, + "learning_rate": 8.8e-06, + "loss": 0.6733917236328125, + "step": 220 + }, + { + "epoch": 0.03692999357739242, + "grad_norm": 2.013750130238611, + "learning_rate": 9.200000000000002e-06, + "loss": 0.6920295715332031, + "step": 230 + }, + { + "epoch": 0.038535645472061654, + "grad_norm": 1.720816299583957, + "learning_rate": 9.600000000000001e-06, + "loss": 0.7383328914642334, + "step": 240 + }, + { + "epoch": 0.040141297366730895, + "grad_norm": 1.819465060907524, + "learning_rate": 1e-05, + "loss": 0.7244165420532227, + "step": 250 + }, + { + "epoch": 0.04174694926140013, + "grad_norm": 1.7972539217653594, + "learning_rate": 9.99993095584273e-06, + "loss": 0.692912483215332, + "step": 260 + }, + { + "epoch": 0.04335260115606936, + "grad_norm": 1.7067105337641253, + "learning_rate": 9.999723825277754e-06, + "loss": 0.660883617401123, + "step": 270 + }, + { + "epoch": 0.0449582530507386, + "grad_norm": 2.125091297709986, + "learning_rate": 9.999378614025538e-06, + "loss": 0.6718532085418701, + "step": 280 + }, + { + "epoch": 0.04656390494540784, + "grad_norm": 2.216748916449176, + "learning_rate": 9.998895331620009e-06, + "loss": 0.6847996711730957, + "step": 290 + }, + { + "epoch": 0.04816955684007707, + "grad_norm": 1.7631756878355385, + "learning_rate": 9.998273991408293e-06, + "loss": 0.6685489177703857, + "step": 300 + }, + { + "epoch": 0.04977520873474631, + "grad_norm": 1.839149164708947, + "learning_rate": 9.997514610550363e-06, + "loss": 0.7628218173980713, + "step": 310 + }, + { + "epoch": 0.05138086062941554, + "grad_norm": 1.856684857256113, + "learning_rate": 9.996617210018536e-06, + "loss": 0.6952354431152343, + "step": 320 + }, + { + "epoch": 0.052986512524084775, + "grad_norm": 1.9416524505459372, + "learning_rate": 9.995581814596923e-06, + "loss": 0.7695009708404541, + "step": 330 + }, + { + "epoch": 0.054592164418754016, + "grad_norm": 1.7934258979637172, + "learning_rate": 9.99440845288072e-06, + "loss": 0.6912702560424805, + "step": 340 + }, + { + "epoch": 0.05619781631342325, + "grad_norm": 1.9412822952421416, + "learning_rate": 9.99309715727544e-06, + "loss": 0.6723281860351562, + "step": 350 + }, + { + "epoch": 0.057803468208092484, + "grad_norm": 1.7740426112390544, + "learning_rate": 9.991647963996001e-06, + "loss": 0.6599285125732421, + "step": 360 + }, + { + "epoch": 0.05940912010276172, + "grad_norm": 1.7505849691964597, + "learning_rate": 9.990060913065735e-06, + "loss": 0.7061182022094726, + "step": 370 + }, + { + "epoch": 0.06101477199743096, + "grad_norm": 1.9101134530734751, + "learning_rate": 9.988336048315278e-06, + "loss": 0.6839250087738037, + "step": 380 + }, + { + "epoch": 0.0626204238921002, + "grad_norm": 1.7094030187030334, + "learning_rate": 9.986473417381366e-06, + "loss": 0.6800565242767334, + "step": 390 + }, + { + "epoch": 0.06422607578676943, + "grad_norm": 1.8014199624305205, + "learning_rate": 9.984473071705512e-06, + "loss": 0.6838696479797364, + "step": 400 + }, + { + "epoch": 0.06422607578676943, + "eval_loss": 0.6409494876861572, + "eval_runtime": 116.1986, + "eval_samples_per_second": 17.41, + "eval_steps_per_second": 4.355, + "eval_token_acc": 0.7858714821779255, + "step": 400 + }, + { + "epoch": 0.06583172768143866, + "grad_norm": 1.8806108814717688, + "learning_rate": 9.982335066532586e-06, + "loss": 0.6559715747833252, + "step": 410 + }, + { + "epoch": 0.0674373795761079, + "grad_norm": 1.9102818068882665, + "learning_rate": 9.980059460909298e-06, + "loss": 0.7044609069824219, + "step": 420 + }, + { + "epoch": 0.06904303147077713, + "grad_norm": 1.770001699523884, + "learning_rate": 9.977646317682553e-06, + "loss": 0.659632396697998, + "step": 430 + }, + { + "epoch": 0.07064868336544637, + "grad_norm": 1.3567863695029754, + "learning_rate": 9.975095703497727e-06, + "loss": 0.6679704666137696, + "step": 440 + }, + { + "epoch": 0.07225433526011561, + "grad_norm": 1.6131966739758206, + "learning_rate": 9.972407688796827e-06, + "loss": 0.7166173934936524, + "step": 450 + }, + { + "epoch": 0.07385998715478484, + "grad_norm": 1.7567050641511104, + "learning_rate": 9.969582347816534e-06, + "loss": 0.6758874893188477, + "step": 460 + }, + { + "epoch": 0.07546563904945408, + "grad_norm": 1.8735720372468838, + "learning_rate": 9.966619758586164e-06, + "loss": 0.70075364112854, + "step": 470 + }, + { + "epoch": 0.07707129094412331, + "grad_norm": 1.6791415879981288, + "learning_rate": 9.963520002925506e-06, + "loss": 0.680471658706665, + "step": 480 + }, + { + "epoch": 0.07867694283879255, + "grad_norm": 2.003529169289507, + "learning_rate": 9.960283166442569e-06, + "loss": 0.6862464904785156, + "step": 490 + }, + { + "epoch": 0.08028259473346179, + "grad_norm": 1.974113049309557, + "learning_rate": 9.956909338531211e-06, + "loss": 0.6817655086517334, + "step": 500 + }, + { + "epoch": 0.08188824662813102, + "grad_norm": 2.0172408497940135, + "learning_rate": 9.953398612368673e-06, + "loss": 0.7146397590637207, + "step": 510 + }, + { + "epoch": 0.08349389852280026, + "grad_norm": 1.7093747574962004, + "learning_rate": 9.949751084913008e-06, + "loss": 0.744438648223877, + "step": 520 + }, + { + "epoch": 0.0850995504174695, + "grad_norm": 1.5003850976474211, + "learning_rate": 9.9459668569004e-06, + "loss": 0.703244972229004, + "step": 530 + }, + { + "epoch": 0.08670520231213873, + "grad_norm": 1.798651846954389, + "learning_rate": 9.942046032842381e-06, + "loss": 0.6592291831970215, + "step": 540 + }, + { + "epoch": 0.08831085420680797, + "grad_norm": 1.6787086638075364, + "learning_rate": 9.937988721022948e-06, + "loss": 0.6808198451995849, + "step": 550 + }, + { + "epoch": 0.0899165061014772, + "grad_norm": 1.9058721108189276, + "learning_rate": 9.933795033495575e-06, + "loss": 0.6961453914642334, + "step": 560 + }, + { + "epoch": 0.09152215799614644, + "grad_norm": 1.9916891886759354, + "learning_rate": 9.929465086080106e-06, + "loss": 0.6704145908355713, + "step": 570 + }, + { + "epoch": 0.09312780989081568, + "grad_norm": 1.91373844820144, + "learning_rate": 9.924998998359571e-06, + "loss": 0.6643758773803711, + "step": 580 + }, + { + "epoch": 0.0947334617854849, + "grad_norm": 1.5237514585691028, + "learning_rate": 9.920396893676875e-06, + "loss": 0.663077449798584, + "step": 590 + }, + { + "epoch": 0.09633911368015415, + "grad_norm": 2.145422792432776, + "learning_rate": 9.915658899131393e-06, + "loss": 0.6999767303466797, + "step": 600 + }, + { + "epoch": 0.09633911368015415, + "eval_loss": 0.6344712972640991, + "eval_runtime": 117.1674, + "eval_samples_per_second": 17.266, + "eval_steps_per_second": 4.319, + "eval_token_acc": 0.7857755434719134, + "step": 600 + }, + { + "epoch": 0.09794476557482337, + "grad_norm": 1.8307593239051754, + "learning_rate": 9.910785145575464e-06, + "loss": 0.6796013355255127, + "step": 610 + }, + { + "epoch": 0.09955041746949261, + "grad_norm": 1.6921064740470149, + "learning_rate": 9.905775767610767e-06, + "loss": 0.6893137931823731, + "step": 620 + }, + { + "epoch": 0.10115606936416185, + "grad_norm": 1.733473142410972, + "learning_rate": 9.900630903584616e-06, + "loss": 0.7370388984680176, + "step": 630 + }, + { + "epoch": 0.10276172125883108, + "grad_norm": 1.743574703106915, + "learning_rate": 9.895350695586133e-06, + "loss": 0.7003519058227539, + "step": 640 + }, + { + "epoch": 0.10436737315350032, + "grad_norm": 1.9204364261217848, + "learning_rate": 9.889935289442318e-06, + "loss": 0.6717376232147216, + "step": 650 + }, + { + "epoch": 0.10597302504816955, + "grad_norm": 1.965527006659715, + "learning_rate": 9.884384834714038e-06, + "loss": 0.6846941471099853, + "step": 660 + }, + { + "epoch": 0.10757867694283879, + "grad_norm": 1.4820962727349916, + "learning_rate": 9.878699484691876e-06, + "loss": 0.6158783435821533, + "step": 670 + }, + { + "epoch": 0.10918432883750803, + "grad_norm": 2.0025449794706103, + "learning_rate": 9.872879396391915e-06, + "loss": 0.7124279022216797, + "step": 680 + }, + { + "epoch": 0.11078998073217726, + "grad_norm": 1.8236979976681365, + "learning_rate": 9.866924730551391e-06, + "loss": 0.6713937759399414, + "step": 690 + }, + { + "epoch": 0.1123956326268465, + "grad_norm": 1.9199756005068356, + "learning_rate": 9.860835651624259e-06, + "loss": 0.6933858871459961, + "step": 700 + }, + { + "epoch": 0.11400128452151574, + "grad_norm": 1.8893458383300172, + "learning_rate": 9.854612327776644e-06, + "loss": 0.7108138084411622, + "step": 710 + }, + { + "epoch": 0.11560693641618497, + "grad_norm": 1.5931503496779096, + "learning_rate": 9.848254930882214e-06, + "loss": 0.6837646961212158, + "step": 720 + }, + { + "epoch": 0.11721258831085421, + "grad_norm": 2.037361191773241, + "learning_rate": 9.841763636517406e-06, + "loss": 0.7361730098724365, + "step": 730 + }, + { + "epoch": 0.11881824020552344, + "grad_norm": 1.6947109413422603, + "learning_rate": 9.835138623956603e-06, + "loss": 0.6639810562133789, + "step": 740 + }, + { + "epoch": 0.12042389210019268, + "grad_norm": 1.5747894809434468, + "learning_rate": 9.828380076167167e-06, + "loss": 0.7038897514343262, + "step": 750 + }, + { + "epoch": 0.12202954399486192, + "grad_norm": 1.7564416225223747, + "learning_rate": 9.821488179804394e-06, + "loss": 0.6923412322998047, + "step": 760 + }, + { + "epoch": 0.12363519588953115, + "grad_norm": 1.9384590291788866, + "learning_rate": 9.814463125206356e-06, + "loss": 0.7407946586608887, + "step": 770 + }, + { + "epoch": 0.1252408477842004, + "grad_norm": 1.75724556040692, + "learning_rate": 9.80730510638864e-06, + "loss": 0.6715410709381103, + "step": 780 + }, + { + "epoch": 0.12684649967886963, + "grad_norm": 2.202752788945079, + "learning_rate": 9.800014321038998e-06, + "loss": 0.7091189384460449, + "step": 790 + }, + { + "epoch": 0.12845215157353887, + "grad_norm": 1.7259836719893948, + "learning_rate": 9.792590970511882e-06, + "loss": 0.6885900020599365, + "step": 800 + }, + { + "epoch": 0.12845215157353887, + "eval_loss": 0.635792076587677, + "eval_runtime": 115.7799, + "eval_samples_per_second": 17.473, + "eval_steps_per_second": 4.37, + "eval_token_acc": 0.7863286509683135, + "step": 800 + }, + { + "epoch": 0.13005780346820808, + "grad_norm": 1.9935003932751265, + "learning_rate": 9.785035259822884e-06, + "loss": 0.6880996227264404, + "step": 810 + }, + { + "epoch": 0.13166345536287732, + "grad_norm": 1.804254347504194, + "learning_rate": 9.777347397643075e-06, + "loss": 0.7218072414398193, + "step": 820 + }, + { + "epoch": 0.13326910725754657, + "grad_norm": 1.778836828004283, + "learning_rate": 9.769527596293242e-06, + "loss": 0.6733824253082276, + "step": 830 + }, + { + "epoch": 0.1348747591522158, + "grad_norm": 1.6016327944206012, + "learning_rate": 9.761576071738023e-06, + "loss": 0.644852590560913, + "step": 840 + }, + { + "epoch": 0.13648041104688505, + "grad_norm": 1.7317265857548507, + "learning_rate": 9.753493043579942e-06, + "loss": 0.6704113006591796, + "step": 850 + }, + { + "epoch": 0.13808606294155426, + "grad_norm": 1.5502607757107583, + "learning_rate": 9.745278735053345e-06, + "loss": 0.684635591506958, + "step": 860 + }, + { + "epoch": 0.1396917148362235, + "grad_norm": 1.6322250587933358, + "learning_rate": 9.736933373018236e-06, + "loss": 0.6296725273132324, + "step": 870 + }, + { + "epoch": 0.14129736673089274, + "grad_norm": 1.6281328633715226, + "learning_rate": 9.728457187954013e-06, + "loss": 0.6954986572265625, + "step": 880 + }, + { + "epoch": 0.14290301862556198, + "grad_norm": 1.6762404325968152, + "learning_rate": 9.719850413953095e-06, + "loss": 0.6957962036132812, + "step": 890 + }, + { + "epoch": 0.14450867052023122, + "grad_norm": 1.5869956357406125, + "learning_rate": 9.711113288714466e-06, + "loss": 0.6982949256896973, + "step": 900 + }, + { + "epoch": 0.14611432241490044, + "grad_norm": 1.6499691327729804, + "learning_rate": 9.702246053537108e-06, + "loss": 0.7158383369445801, + "step": 910 + }, + { + "epoch": 0.14771997430956968, + "grad_norm": 1.66525937964079, + "learning_rate": 9.69324895331333e-06, + "loss": 0.7226263046264648, + "step": 920 + }, + { + "epoch": 0.14932562620423892, + "grad_norm": 1.5423485440185492, + "learning_rate": 9.684122236522014e-06, + "loss": 0.6935669898986816, + "step": 930 + }, + { + "epoch": 0.15093127809890816, + "grad_norm": 2.1327741851835467, + "learning_rate": 9.674866155221745e-06, + "loss": 0.6882006168365479, + "step": 940 + }, + { + "epoch": 0.1525369299935774, + "grad_norm": 2.1405917646231063, + "learning_rate": 9.665480965043862e-06, + "loss": 0.6898535251617431, + "step": 950 + }, + { + "epoch": 0.15414258188824662, + "grad_norm": 1.5835812345374503, + "learning_rate": 9.655966925185381e-06, + "loss": 0.6587895393371582, + "step": 960 + }, + { + "epoch": 0.15574823378291586, + "grad_norm": 1.5996285353746509, + "learning_rate": 9.646324298401849e-06, + "loss": 0.7359085083007812, + "step": 970 + }, + { + "epoch": 0.1573538856775851, + "grad_norm": 1.9180234912666367, + "learning_rate": 9.636553351000077e-06, + "loss": 0.7528255462646485, + "step": 980 + }, + { + "epoch": 0.15895953757225434, + "grad_norm": 1.7126844032545163, + "learning_rate": 9.626654352830801e-06, + "loss": 0.667484188079834, + "step": 990 + }, + { + "epoch": 0.16056518946692358, + "grad_norm": 1.9579386418721534, + "learning_rate": 9.616627577281217e-06, + "loss": 0.7283291816711426, + "step": 1000 + }, + { + "epoch": 0.16056518946692358, + "eval_loss": 0.6324719786643982, + "eval_runtime": 117.4402, + "eval_samples_per_second": 17.226, + "eval_steps_per_second": 4.309, + "eval_token_acc": 0.7870844811217654, + "step": 1000 + }, + { + "epoch": 0.1621708413615928, + "grad_norm": 2.124330762824706, + "learning_rate": 9.606473301267427e-06, + "loss": 0.7245466232299804, + "step": 1010 + }, + { + "epoch": 0.16377649325626203, + "grad_norm": 1.7661184806882184, + "learning_rate": 9.59619180522681e-06, + "loss": 0.6639774322509766, + "step": 1020 + }, + { + "epoch": 0.16538214515093128, + "grad_norm": 1.8821025912253, + "learning_rate": 9.585783373110248e-06, + "loss": 0.6982080459594726, + "step": 1030 + }, + { + "epoch": 0.16698779704560052, + "grad_norm": 1.7845462984409228, + "learning_rate": 9.575248292374322e-06, + "loss": 0.6804990768432617, + "step": 1040 + }, + { + "epoch": 0.16859344894026976, + "grad_norm": 1.777143469617724, + "learning_rate": 9.564586853973332e-06, + "loss": 0.7265186309814453, + "step": 1050 + }, + { + "epoch": 0.170199100834939, + "grad_norm": 1.6943629817215575, + "learning_rate": 9.553799352351293e-06, + "loss": 0.6963564872741699, + "step": 1060 + }, + { + "epoch": 0.1718047527296082, + "grad_norm": 1.851666049439146, + "learning_rate": 9.54288608543379e-06, + "loss": 0.6982178688049316, + "step": 1070 + }, + { + "epoch": 0.17341040462427745, + "grad_norm": 1.7784015364121233, + "learning_rate": 9.531847354619745e-06, + "loss": 0.6448168277740478, + "step": 1080 + }, + { + "epoch": 0.1750160565189467, + "grad_norm": 1.6328432019971293, + "learning_rate": 9.52068346477311e-06, + "loss": 0.689232587814331, + "step": 1090 + }, + { + "epoch": 0.17662170841361594, + "grad_norm": 1.6395747590691885, + "learning_rate": 9.509394724214428e-06, + "loss": 0.6840867042541504, + "step": 1100 + }, + { + "epoch": 0.17822736030828518, + "grad_norm": 1.7013838480613497, + "learning_rate": 9.497981444712332e-06, + "loss": 0.7155774593353271, + "step": 1110 + }, + { + "epoch": 0.1798330122029544, + "grad_norm": 1.800392000928992, + "learning_rate": 9.486443941474928e-06, + "loss": 0.7150219917297364, + "step": 1120 + }, + { + "epoch": 0.18143866409762363, + "grad_norm": 1.8225619060168265, + "learning_rate": 9.47478253314109e-06, + "loss": 0.7047001838684082, + "step": 1130 + }, + { + "epoch": 0.18304431599229287, + "grad_norm": 1.6921831025517564, + "learning_rate": 9.462997541771664e-06, + "loss": 0.691263484954834, + "step": 1140 + }, + { + "epoch": 0.1846499678869621, + "grad_norm": 1.5988960134584689, + "learning_rate": 9.451089292840569e-06, + "loss": 0.6462996482849122, + "step": 1150 + }, + { + "epoch": 0.18625561978163135, + "grad_norm": 1.6624610693181532, + "learning_rate": 9.439058115225808e-06, + "loss": 0.7005721092224121, + "step": 1160 + }, + { + "epoch": 0.18786127167630057, + "grad_norm": 1.7408908224585418, + "learning_rate": 9.42690434120039e-06, + "loss": 0.6465234279632568, + "step": 1170 + }, + { + "epoch": 0.1894669235709698, + "grad_norm": 2.0230964704240093, + "learning_rate": 9.414628306423148e-06, + "loss": 0.7096317291259766, + "step": 1180 + }, + { + "epoch": 0.19107257546563905, + "grad_norm": 1.5438413479774158, + "learning_rate": 9.402230349929475e-06, + "loss": 0.687491512298584, + "step": 1190 + }, + { + "epoch": 0.1926782273603083, + "grad_norm": 1.6673847990845716, + "learning_rate": 9.389710814121951e-06, + "loss": 0.7050128936767578, + "step": 1200 + }, + { + "epoch": 0.1926782273603083, + "eval_loss": 0.6267312169075012, + "eval_runtime": 116.5867, + "eval_samples_per_second": 17.352, + "eval_steps_per_second": 4.34, + "eval_token_acc": 0.7883133308083379, + "step": 1200 + }, + { + "epoch": 0.19428387925497753, + "grad_norm": 1.9036108770716342, + "learning_rate": 9.377070044760899e-06, + "loss": 0.6793604850769043, + "step": 1210 + }, + { + "epoch": 0.19588953114964675, + "grad_norm": 1.7574024142219666, + "learning_rate": 9.364308390954823e-06, + "loss": 0.6633443355560302, + "step": 1220 + }, + { + "epoch": 0.197495183044316, + "grad_norm": 1.8418482772911668, + "learning_rate": 9.351426205150778e-06, + "loss": 0.6838456153869629, + "step": 1230 + }, + { + "epoch": 0.19910083493898523, + "grad_norm": 1.780986059541626, + "learning_rate": 9.338423843124627e-06, + "loss": 0.7096598625183106, + "step": 1240 + }, + { + "epoch": 0.20070648683365447, + "grad_norm": 1.467140963611623, + "learning_rate": 9.325301663971222e-06, + "loss": 0.7107316017150879, + "step": 1250 + }, + { + "epoch": 0.2023121387283237, + "grad_norm": 2.1182017546402827, + "learning_rate": 9.312060030094487e-06, + "loss": 0.714271354675293, + "step": 1260 + }, + { + "epoch": 0.20391779062299292, + "grad_norm": 1.8241480529102874, + "learning_rate": 9.298699307197398e-06, + "loss": 0.6439716339111328, + "step": 1270 + }, + { + "epoch": 0.20552344251766216, + "grad_norm": 2.074514301094621, + "learning_rate": 9.2852198642719e-06, + "loss": 0.6223122119903565, + "step": 1280 + }, + { + "epoch": 0.2071290944123314, + "grad_norm": 1.6495995877234806, + "learning_rate": 9.271622073588699e-06, + "loss": 0.6849968910217286, + "step": 1290 + }, + { + "epoch": 0.20873474630700065, + "grad_norm": 1.5842560579717333, + "learning_rate": 9.257906310686999e-06, + "loss": 0.6660888671875, + "step": 1300 + }, + { + "epoch": 0.2103403982016699, + "grad_norm": 1.7025944783720839, + "learning_rate": 9.244072954364116e-06, + "loss": 0.689777946472168, + "step": 1310 + }, + { + "epoch": 0.2119460500963391, + "grad_norm": 1.6143671464056364, + "learning_rate": 9.23012238666502e-06, + "loss": 0.6809407234191894, + "step": 1320 + }, + { + "epoch": 0.21355170199100834, + "grad_norm": 1.750371013085034, + "learning_rate": 9.216054992871787e-06, + "loss": 0.6946802616119385, + "step": 1330 + }, + { + "epoch": 0.21515735388567758, + "grad_norm": 1.6296247737445233, + "learning_rate": 9.201871161492957e-06, + "loss": 0.669765043258667, + "step": 1340 + }, + { + "epoch": 0.21676300578034682, + "grad_norm": 1.589845342046221, + "learning_rate": 9.187571284252806e-06, + "loss": 0.6649596214294433, + "step": 1350 + }, + { + "epoch": 0.21836865767501606, + "grad_norm": 1.8048450397597646, + "learning_rate": 9.17315575608052e-06, + "loss": 0.6881882667541503, + "step": 1360 + }, + { + "epoch": 0.2199743095696853, + "grad_norm": 1.8342938642969528, + "learning_rate": 9.158624975099299e-06, + "loss": 0.7048683166503906, + "step": 1370 + }, + { + "epoch": 0.22157996146435452, + "grad_norm": 1.6243711900499107, + "learning_rate": 9.143979342615354e-06, + "loss": 0.7189963817596435, + "step": 1380 + }, + { + "epoch": 0.22318561335902376, + "grad_norm": 1.8230485495669633, + "learning_rate": 9.129219263106825e-06, + "loss": 0.683738899230957, + "step": 1390 + }, + { + "epoch": 0.224791265253693, + "grad_norm": 1.5695908073726854, + "learning_rate": 9.11434514421261e-06, + "loss": 0.6511239051818848, + "step": 1400 + }, + { + "epoch": 0.224791265253693, + "eval_loss": 0.6287193894386292, + "eval_runtime": 116.9521, + "eval_samples_per_second": 17.298, + "eval_steps_per_second": 4.327, + "eval_token_acc": 0.7881156136489912, + "step": 1400 + }, + { + "epoch": 0.22639691714836224, + "grad_norm": 1.4800767435988433, + "learning_rate": 9.099357396721117e-06, + "loss": 0.6380510330200195, + "step": 1410 + }, + { + "epoch": 0.22800256904303148, + "grad_norm": 1.9797567309083863, + "learning_rate": 9.084256434558898e-06, + "loss": 0.6463112831115723, + "step": 1420 + }, + { + "epoch": 0.2296082209377007, + "grad_norm": 1.7618381398848315, + "learning_rate": 9.069042674779238e-06, + "loss": 0.6677730560302735, + "step": 1430 + }, + { + "epoch": 0.23121387283236994, + "grad_norm": 1.6958553368070852, + "learning_rate": 9.053716537550627e-06, + "loss": 0.6688085556030273, + "step": 1440 + }, + { + "epoch": 0.23281952472703918, + "grad_norm": 1.4712510577159659, + "learning_rate": 9.038278446145155e-06, + "loss": 0.7256585121154785, + "step": 1450 + }, + { + "epoch": 0.23442517662170842, + "grad_norm": 1.5054612032282009, + "learning_rate": 9.022728826926825e-06, + "loss": 0.6773605823516846, + "step": 1460 + }, + { + "epoch": 0.23603082851637766, + "grad_norm": 1.9511000712849178, + "learning_rate": 9.007068109339783e-06, + "loss": 0.6480167388916016, + "step": 1470 + }, + { + "epoch": 0.23763648041104687, + "grad_norm": 1.8030816945625912, + "learning_rate": 8.991296725896449e-06, + "loss": 0.706729507446289, + "step": 1480 + }, + { + "epoch": 0.23924213230571612, + "grad_norm": 1.7469991434387138, + "learning_rate": 8.975415112165566e-06, + "loss": 0.697143840789795, + "step": 1490 + }, + { + "epoch": 0.24084778420038536, + "grad_norm": 1.4639433653164637, + "learning_rate": 8.959423706760197e-06, + "loss": 0.6564635276794434, + "step": 1500 + }, + { + "epoch": 0.2424534360950546, + "grad_norm": 1.829707419585373, + "learning_rate": 8.943322951325583e-06, + "loss": 0.6853602409362793, + "step": 1510 + }, + { + "epoch": 0.24405908798972384, + "grad_norm": 1.6079876857975004, + "learning_rate": 8.927113290526961e-06, + "loss": 0.64951171875, + "step": 1520 + }, + { + "epoch": 0.24566473988439305, + "grad_norm": 1.962740635067071, + "learning_rate": 8.910795172037278e-06, + "loss": 0.6620121002197266, + "step": 1530 + }, + { + "epoch": 0.2472703917790623, + "grad_norm": 1.816612343146698, + "learning_rate": 8.894369046524829e-06, + "loss": 0.6383034706115722, + "step": 1540 + }, + { + "epoch": 0.24887604367373153, + "grad_norm": 1.58678618072732, + "learning_rate": 8.877835367640813e-06, + "loss": 0.6568033218383789, + "step": 1550 + }, + { + "epoch": 0.2504816955684008, + "grad_norm": 1.5302435443839166, + "learning_rate": 8.861194592006798e-06, + "loss": 0.6800635814666748, + "step": 1560 + }, + { + "epoch": 0.25208734746307, + "grad_norm": 1.8654625116067247, + "learning_rate": 8.844447179202119e-06, + "loss": 0.6741923809051513, + "step": 1570 + }, + { + "epoch": 0.25369299935773926, + "grad_norm": 1.5369005787200434, + "learning_rate": 8.827593591751172e-06, + "loss": 0.7129825592041016, + "step": 1580 + }, + { + "epoch": 0.25529865125240847, + "grad_norm": 1.7043874051543522, + "learning_rate": 8.810634295110661e-06, + "loss": 0.660031795501709, + "step": 1590 + }, + { + "epoch": 0.25690430314707774, + "grad_norm": 1.6582761173205205, + "learning_rate": 8.793569757656718e-06, + "loss": 0.6948627471923828, + "step": 1600 + }, + { + "epoch": 0.25690430314707774, + "eval_loss": 0.6219077706336975, + "eval_runtime": 116.5993, + "eval_samples_per_second": 17.35, + "eval_steps_per_second": 4.34, + "eval_token_acc": 0.7890674924625547, + "step": 1600 + }, + { + "epoch": 0.25850995504174695, + "grad_norm": 1.8529510991569655, + "learning_rate": 8.77640045067199e-06, + "loss": 0.6722494602203369, + "step": 1610 + }, + { + "epoch": 0.26011560693641617, + "grad_norm": 1.9357141571987, + "learning_rate": 8.759126848332608e-06, + "loss": 0.6848864078521728, + "step": 1620 + }, + { + "epoch": 0.26172125883108543, + "grad_norm": 1.6096339964942243, + "learning_rate": 8.7417494276951e-06, + "loss": 0.6899352073669434, + "step": 1630 + }, + { + "epoch": 0.26332691072575465, + "grad_norm": 1.7064573876372433, + "learning_rate": 8.724268668683207e-06, + "loss": 0.7040124893188476, + "step": 1640 + }, + { + "epoch": 0.2649325626204239, + "grad_norm": 1.7909548168508456, + "learning_rate": 8.706685054074644e-06, + "loss": 0.6199952125549316, + "step": 1650 + }, + { + "epoch": 0.26653821451509313, + "grad_norm": 1.8418407360742293, + "learning_rate": 8.688999069487749e-06, + "loss": 0.6850284576416016, + "step": 1660 + }, + { + "epoch": 0.26814386640976234, + "grad_norm": 1.5253465102049333, + "learning_rate": 8.671211203368083e-06, + "loss": 0.6807417869567871, + "step": 1670 + }, + { + "epoch": 0.2697495183044316, + "grad_norm": 1.6746489328318919, + "learning_rate": 8.653321946974939e-06, + "loss": 0.6941102981567383, + "step": 1680 + }, + { + "epoch": 0.2713551701991008, + "grad_norm": 1.4937171287944617, + "learning_rate": 8.635331794367766e-06, + "loss": 0.6537214279174804, + "step": 1690 + }, + { + "epoch": 0.2729608220937701, + "grad_norm": 1.6787435216248838, + "learning_rate": 8.617241242392535e-06, + "loss": 0.6437517642974854, + "step": 1700 + }, + { + "epoch": 0.2745664739884393, + "grad_norm": 2.052971205655963, + "learning_rate": 8.599050790668016e-06, + "loss": 0.6665167331695556, + "step": 1710 + }, + { + "epoch": 0.2761721258831085, + "grad_norm": 1.737556496234147, + "learning_rate": 8.580760941571968e-06, + "loss": 0.7035176277160644, + "step": 1720 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 1.7432018470950634, + "learning_rate": 8.56237220022728e-06, + "loss": 0.7075429916381836, + "step": 1730 + }, + { + "epoch": 0.279383429672447, + "grad_norm": 1.860416768706513, + "learning_rate": 8.543885074488012e-06, + "loss": 0.6828268051147461, + "step": 1740 + }, + { + "epoch": 0.2809890815671163, + "grad_norm": 1.7610785827294768, + "learning_rate": 8.525300074925371e-06, + "loss": 0.6819411277770996, + "step": 1750 + }, + { + "epoch": 0.2825947334617855, + "grad_norm": 1.804081843028565, + "learning_rate": 8.50661771481361e-06, + "loss": 0.6932484626770019, + "step": 1760 + }, + { + "epoch": 0.2842003853564547, + "grad_norm": 1.6441985906441237, + "learning_rate": 8.48783851011585e-06, + "loss": 0.6582047462463378, + "step": 1770 + }, + { + "epoch": 0.28580603725112397, + "grad_norm": 1.7329173754267528, + "learning_rate": 8.468962979469841e-06, + "loss": 0.7075380325317383, + "step": 1780 + }, + { + "epoch": 0.2874116891457932, + "grad_norm": 1.4693447118521001, + "learning_rate": 8.449991644173624e-06, + "loss": 0.6702975273132324, + "step": 1790 + }, + { + "epoch": 0.28901734104046245, + "grad_norm": 1.6100326348735192, + "learning_rate": 8.43092502817114e-06, + "loss": 0.6638934135437011, + "step": 1800 + }, + { + "epoch": 0.28901734104046245, + "eval_loss": 0.6190060973167419, + "eval_runtime": 115.7311, + "eval_samples_per_second": 17.48, + "eval_steps_per_second": 4.372, + "eval_token_acc": 0.7904098000971067, + "step": 1800 + }, + { + "epoch": 0.29062299293513166, + "grad_norm": 1.8046113407221105, + "learning_rate": 8.411763658037764e-06, + "loss": 0.6865541458129882, + "step": 1810 + }, + { + "epoch": 0.2922286448298009, + "grad_norm": 1.66384985336402, + "learning_rate": 8.392508062965758e-06, + "loss": 0.7061246395111084, + "step": 1820 + }, + { + "epoch": 0.29383429672447015, + "grad_norm": 1.4509377462492312, + "learning_rate": 8.373158774749654e-06, + "loss": 0.6638317584991456, + "step": 1830 + }, + { + "epoch": 0.29543994861913936, + "grad_norm": 1.5250629360772658, + "learning_rate": 8.353716327771572e-06, + "loss": 0.6382759571075439, + "step": 1840 + }, + { + "epoch": 0.2970456005138086, + "grad_norm": 1.7177671490174162, + "learning_rate": 8.33418125898646e-06, + "loss": 0.675161075592041, + "step": 1850 + }, + { + "epoch": 0.29865125240847784, + "grad_norm": 1.7352785342373054, + "learning_rate": 8.314554107907262e-06, + "loss": 0.6799586772918701, + "step": 1860 + }, + { + "epoch": 0.30025690430314705, + "grad_norm": 1.8297976880260405, + "learning_rate": 8.294835416590019e-06, + "loss": 0.6495511054992675, + "step": 1870 + }, + { + "epoch": 0.3018625561978163, + "grad_norm": 1.7771603972640657, + "learning_rate": 8.275025729618902e-06, + "loss": 0.6970042705535888, + "step": 1880 + }, + { + "epoch": 0.30346820809248554, + "grad_norm": 1.692739401160697, + "learning_rate": 8.255125594091169e-06, + "loss": 0.6314344882965088, + "step": 1890 + }, + { + "epoch": 0.3050738599871548, + "grad_norm": 1.5647739461600825, + "learning_rate": 8.235135559602055e-06, + "loss": 0.6312388896942138, + "step": 1900 + }, + { + "epoch": 0.306679511881824, + "grad_norm": 1.8745989880906309, + "learning_rate": 8.21505617822959e-06, + "loss": 0.7160224437713623, + "step": 1910 + }, + { + "epoch": 0.30828516377649323, + "grad_norm": 1.794790643737546, + "learning_rate": 8.194888004519365e-06, + "loss": 0.7057693481445313, + "step": 1920 + }, + { + "epoch": 0.3098908156711625, + "grad_norm": 1.5750000095656258, + "learning_rate": 8.1746315954692e-06, + "loss": 0.6810090065002441, + "step": 1930 + }, + { + "epoch": 0.3114964675658317, + "grad_norm": 1.7006910513410733, + "learning_rate": 8.154287510513773e-06, + "loss": 0.6751882553100585, + "step": 1940 + }, + { + "epoch": 0.313102119460501, + "grad_norm": 1.5282577764518495, + "learning_rate": 8.133856311509165e-06, + "loss": 0.6727633476257324, + "step": 1950 + }, + { + "epoch": 0.3147077713551702, + "grad_norm": 1.5855730147878353, + "learning_rate": 8.113338562717341e-06, + "loss": 0.6358974933624267, + "step": 1960 + }, + { + "epoch": 0.3163134232498394, + "grad_norm": 1.624282628411416, + "learning_rate": 8.092734830790575e-06, + "loss": 0.634096622467041, + "step": 1970 + }, + { + "epoch": 0.3179190751445087, + "grad_norm": 1.8557574949773143, + "learning_rate": 8.072045684755783e-06, + "loss": 0.673964548110962, + "step": 1980 + }, + { + "epoch": 0.3195247270391779, + "grad_norm": 1.6098240624386007, + "learning_rate": 8.051271695998832e-06, + "loss": 0.6814305782318115, + "step": 1990 + }, + { + "epoch": 0.32113037893384716, + "grad_norm": 1.8638715234892917, + "learning_rate": 8.03041343824874e-06, + "loss": 0.7079939842224121, + "step": 2000 + }, + { + "epoch": 0.32113037893384716, + "eval_loss": 0.6177350878715515, + "eval_runtime": 115.694, + "eval_samples_per_second": 17.486, + "eval_steps_per_second": 4.374, + "eval_token_acc": 0.7912348729688108, + "step": 2000 + }, + { + "epoch": 0.3227360308285164, + "grad_norm": 1.6815059735101043, + "learning_rate": 8.009471487561837e-06, + "loss": 0.6493409156799317, + "step": 2010 + }, + { + "epoch": 0.3243416827231856, + "grad_norm": 2.1534893724823303, + "learning_rate": 7.988446422305857e-06, + "loss": 0.7218660354614258, + "step": 2020 + }, + { + "epoch": 0.32594733461785486, + "grad_norm": 1.4898609880434688, + "learning_rate": 7.967338823143967e-06, + "loss": 0.6944834232330322, + "step": 2030 + }, + { + "epoch": 0.32755298651252407, + "grad_norm": 1.7981374103572556, + "learning_rate": 7.946149273018723e-06, + "loss": 0.6680311679840087, + "step": 2040 + }, + { + "epoch": 0.32915863840719334, + "grad_norm": 1.7153248644552248, + "learning_rate": 7.92487835713598e-06, + "loss": 0.6817049503326416, + "step": 2050 + }, + { + "epoch": 0.33076429030186255, + "grad_norm": 1.5225633936289904, + "learning_rate": 7.903526662948721e-06, + "loss": 0.659334945678711, + "step": 2060 + }, + { + "epoch": 0.33236994219653176, + "grad_norm": 1.870644237710989, + "learning_rate": 7.882094780140838e-06, + "loss": 0.6837968826293945, + "step": 2070 + }, + { + "epoch": 0.33397559409120103, + "grad_norm": 1.6975397564317007, + "learning_rate": 7.860583300610849e-06, + "loss": 0.6264148235321045, + "step": 2080 + }, + { + "epoch": 0.33558124598587025, + "grad_norm": 1.5490837406732572, + "learning_rate": 7.838992818455542e-06, + "loss": 0.6516887187957764, + "step": 2090 + }, + { + "epoch": 0.3371868978805395, + "grad_norm": 1.8074910342864772, + "learning_rate": 7.817323929953575e-06, + "loss": 0.6940847873687744, + "step": 2100 + }, + { + "epoch": 0.33879254977520873, + "grad_norm": 1.7586469146797696, + "learning_rate": 7.795577233549006e-06, + "loss": 0.6694756031036377, + "step": 2110 + }, + { + "epoch": 0.340398201669878, + "grad_norm": 1.6458423161196865, + "learning_rate": 7.773753329834767e-06, + "loss": 0.6358282566070557, + "step": 2120 + }, + { + "epoch": 0.3420038535645472, + "grad_norm": 1.5559151397357136, + "learning_rate": 7.751852821536073e-06, + "loss": 0.7031454086303711, + "step": 2130 + }, + { + "epoch": 0.3436095054592164, + "grad_norm": 1.8641173427315336, + "learning_rate": 7.729876313493781e-06, + "loss": 0.6741362094879151, + "step": 2140 + }, + { + "epoch": 0.3452151573538857, + "grad_norm": 1.8782029134936515, + "learning_rate": 7.70782441264768e-06, + "loss": 0.7104865550994873, + "step": 2150 + }, + { + "epoch": 0.3468208092485549, + "grad_norm": 1.663908552844927, + "learning_rate": 7.68569772801974e-06, + "loss": 0.6510393619537354, + "step": 2160 + }, + { + "epoch": 0.3484264611432242, + "grad_norm": 1.9798217750009868, + "learning_rate": 7.663496870697267e-06, + "loss": 0.6797806739807128, + "step": 2170 + }, + { + "epoch": 0.3500321130378934, + "grad_norm": 1.6234497136185502, + "learning_rate": 7.641222453816064e-06, + "loss": 0.6498265266418457, + "step": 2180 + }, + { + "epoch": 0.3516377649325626, + "grad_norm": 1.7308643473117888, + "learning_rate": 7.618875092543467e-06, + "loss": 0.6692814826965332, + "step": 2190 + }, + { + "epoch": 0.35324341682723187, + "grad_norm": 1.805844797993927, + "learning_rate": 7.596455404061365e-06, + "loss": 0.6444426536560058, + "step": 2200 + }, + { + "epoch": 0.35324341682723187, + "eval_loss": 0.6147180199623108, + "eval_runtime": 117.4034, + "eval_samples_per_second": 17.231, + "eval_steps_per_second": 4.31, + "eval_token_acc": 0.7917295829919861, + "step": 2200 + }, + { + "epoch": 0.3548490687219011, + "grad_norm": 1.818860826365226, + "learning_rate": 7.5739640075491546e-06, + "loss": 0.6568302154541016, + "step": 2210 + }, + { + "epoch": 0.35645472061657035, + "grad_norm": 1.8515151048630425, + "learning_rate": 7.551401524166646e-06, + "loss": 0.6514512062072754, + "step": 2220 + }, + { + "epoch": 0.35806037251123957, + "grad_norm": 1.5941745195897792, + "learning_rate": 7.5287685770369e-06, + "loss": 0.6203208446502686, + "step": 2230 + }, + { + "epoch": 0.3596660244059088, + "grad_norm": 1.9079336044757396, + "learning_rate": 7.506065791229018e-06, + "loss": 0.6402321338653565, + "step": 2240 + }, + { + "epoch": 0.36127167630057805, + "grad_norm": 1.8381168124624487, + "learning_rate": 7.48329379374089e-06, + "loss": 0.6790849685668945, + "step": 2250 + }, + { + "epoch": 0.36287732819524726, + "grad_norm": 1.9481710547040232, + "learning_rate": 7.460453213481862e-06, + "loss": 0.6818710803985596, + "step": 2260 + }, + { + "epoch": 0.36448298008991653, + "grad_norm": 1.6938215893010562, + "learning_rate": 7.437544681255383e-06, + "loss": 0.6768095970153809, + "step": 2270 + }, + { + "epoch": 0.36608863198458574, + "grad_norm": 1.7075992895641596, + "learning_rate": 7.414568829741572e-06, + "loss": 0.6130049228668213, + "step": 2280 + }, + { + "epoch": 0.36769428387925496, + "grad_norm": 1.7491228083237265, + "learning_rate": 7.3915262934797525e-06, + "loss": 0.662748908996582, + "step": 2290 + }, + { + "epoch": 0.3692999357739242, + "grad_norm": 1.6584152969999681, + "learning_rate": 7.368417708850923e-06, + "loss": 0.7056490898132324, + "step": 2300 + }, + { + "epoch": 0.37090558766859344, + "grad_norm": 1.7027901094047575, + "learning_rate": 7.3452437140601855e-06, + "loss": 0.7262134552001953, + "step": 2310 + }, + { + "epoch": 0.3725112395632627, + "grad_norm": 1.6192575463774184, + "learning_rate": 7.322004949119114e-06, + "loss": 0.6772657871246338, + "step": 2320 + }, + { + "epoch": 0.3741168914579319, + "grad_norm": 1.5966299076510366, + "learning_rate": 7.298702055828086e-06, + "loss": 0.6767099857330322, + "step": 2330 + }, + { + "epoch": 0.37572254335260113, + "grad_norm": 1.6940198781585711, + "learning_rate": 7.275335677758553e-06, + "loss": 0.6707070350646973, + "step": 2340 + }, + { + "epoch": 0.3773281952472704, + "grad_norm": 1.3801221057150768, + "learning_rate": 7.251906460235268e-06, + "loss": 0.6307042121887207, + "step": 2350 + }, + { + "epoch": 0.3789338471419396, + "grad_norm": 1.5598911491306946, + "learning_rate": 7.228415050318463e-06, + "loss": 0.7124890804290771, + "step": 2360 + }, + { + "epoch": 0.3805394990366089, + "grad_norm": 1.8708516506090664, + "learning_rate": 7.204862096785978e-06, + "loss": 0.6433597087860108, + "step": 2370 + }, + { + "epoch": 0.3821451509312781, + "grad_norm": 1.706542614787211, + "learning_rate": 7.181248250115346e-06, + "loss": 0.6750481128692627, + "step": 2380 + }, + { + "epoch": 0.3837508028259473, + "grad_norm": 1.719256978853715, + "learning_rate": 7.1575741624658215e-06, + "loss": 0.6577974319458008, + "step": 2390 + }, + { + "epoch": 0.3853564547206166, + "grad_norm": 1.7008700295216381, + "learning_rate": 7.1338404876603784e-06, + "loss": 0.6774620532989502, + "step": 2400 + }, + { + "epoch": 0.3853564547206166, + "eval_loss": 0.6114417314529419, + "eval_runtime": 117.9272, + "eval_samples_per_second": 17.155, + "eval_steps_per_second": 4.291, + "eval_token_acc": 0.7923302427165837, + "step": 2400 + }, + { + "epoch": 0.3869621066152858, + "grad_norm": 1.570715977392802, + "learning_rate": 7.110047881167647e-06, + "loss": 0.6567262649536133, + "step": 2410 + }, + { + "epoch": 0.38856775850995506, + "grad_norm": 1.7488962726005084, + "learning_rate": 7.086197000083812e-06, + "loss": 0.6914029598236084, + "step": 2420 + }, + { + "epoch": 0.3901734104046243, + "grad_norm": 1.7579994392685563, + "learning_rate": 7.0622885031144685e-06, + "loss": 0.674524450302124, + "step": 2430 + }, + { + "epoch": 0.3917790622992935, + "grad_norm": 1.9538185949651201, + "learning_rate": 7.038323050556426e-06, + "loss": 0.7166782855987549, + "step": 2440 + }, + { + "epoch": 0.39338471419396276, + "grad_norm": 1.616334748220648, + "learning_rate": 7.014301304279476e-06, + "loss": 0.6658863067626953, + "step": 2450 + }, + { + "epoch": 0.394990366088632, + "grad_norm": 2.1028001883855563, + "learning_rate": 6.990223927708107e-06, + "loss": 0.6730957984924316, + "step": 2460 + }, + { + "epoch": 0.39659601798330124, + "grad_norm": 1.9382549699960543, + "learning_rate": 6.966091585803191e-06, + "loss": 0.6466156482696533, + "step": 2470 + }, + { + "epoch": 0.39820166987797045, + "grad_norm": 1.874004069716299, + "learning_rate": 6.94190494504361e-06, + "loss": 0.6450675010681153, + "step": 2480 + }, + { + "epoch": 0.39980732177263967, + "grad_norm": 1.825141725895593, + "learning_rate": 6.917664673407858e-06, + "loss": 0.6538877487182617, + "step": 2490 + }, + { + "epoch": 0.40141297366730894, + "grad_norm": 1.6769353130289677, + "learning_rate": 6.893371440355585e-06, + "loss": 0.6547831535339356, + "step": 2500 + }, + { + "epoch": 0.40301862556197815, + "grad_norm": 1.718505237274443, + "learning_rate": 6.8690259168091115e-06, + "loss": 0.7072465896606446, + "step": 2510 + }, + { + "epoch": 0.4046242774566474, + "grad_norm": 1.6116686542587728, + "learning_rate": 6.8446287751349e-06, + "loss": 0.6458945274353027, + "step": 2520 + }, + { + "epoch": 0.40622992935131663, + "grad_norm": 1.6485734517298922, + "learning_rate": 6.820180689124984e-06, + "loss": 0.634568452835083, + "step": 2530 + }, + { + "epoch": 0.40783558124598585, + "grad_norm": 1.693905081233172, + "learning_rate": 6.795682333978365e-06, + "loss": 0.6384255409240722, + "step": 2540 + }, + { + "epoch": 0.4094412331406551, + "grad_norm": 1.8429999773263868, + "learning_rate": 6.771134386282355e-06, + "loss": 0.6711873054504395, + "step": 2550 + }, + { + "epoch": 0.4110468850353243, + "grad_norm": 1.5334617046393333, + "learning_rate": 6.7465375239939e-06, + "loss": 0.6835154533386231, + "step": 2560 + }, + { + "epoch": 0.4126525369299936, + "grad_norm": 1.8079819637608645, + "learning_rate": 6.721892426420851e-06, + "loss": 0.6307646751403808, + "step": 2570 + }, + { + "epoch": 0.4142581888246628, + "grad_norm": 1.6979554296520882, + "learning_rate": 6.697199774203203e-06, + "loss": 0.6037578582763672, + "step": 2580 + }, + { + "epoch": 0.415863840719332, + "grad_norm": 1.8244704576295199, + "learning_rate": 6.6724602492943035e-06, + "loss": 0.6572129726409912, + "step": 2590 + }, + { + "epoch": 0.4174694926140013, + "grad_norm": 1.6572633090594417, + "learning_rate": 6.64767453494201e-06, + "loss": 0.6604421615600586, + "step": 2600 + }, + { + "epoch": 0.4174694926140013, + "eval_loss": 0.6077189445495605, + "eval_runtime": 116.5488, + "eval_samples_per_second": 17.358, + "eval_steps_per_second": 4.342, + "eval_token_acc": 0.7931561498379053, + "step": 2600 + }, + { + "epoch": 0.4190751445086705, + "grad_norm": 1.9987895534078324, + "learning_rate": 6.6228433156698295e-06, + "loss": 0.6792376518249512, + "step": 2610 + }, + { + "epoch": 0.4206807964033398, + "grad_norm": 1.7158981483927298, + "learning_rate": 6.597967277258003e-06, + "loss": 0.6651222229003906, + "step": 2620 + }, + { + "epoch": 0.422286448298009, + "grad_norm": 1.7697895779850215, + "learning_rate": 6.573047106724574e-06, + "loss": 0.6623071670532227, + "step": 2630 + }, + { + "epoch": 0.4238921001926782, + "grad_norm": 2.126926145770739, + "learning_rate": 6.548083492306413e-06, + "loss": 0.6466917037963867, + "step": 2640 + }, + { + "epoch": 0.42549775208734747, + "grad_norm": 1.4780847013631149, + "learning_rate": 6.523077123440207e-06, + "loss": 0.6583863258361816, + "step": 2650 + }, + { + "epoch": 0.4271034039820167, + "grad_norm": 1.791311406001526, + "learning_rate": 6.498028690743422e-06, + "loss": 0.6781702041625977, + "step": 2660 + }, + { + "epoch": 0.42870905587668595, + "grad_norm": 1.8418453284917216, + "learning_rate": 6.472938885995229e-06, + "loss": 0.615170955657959, + "step": 2670 + }, + { + "epoch": 0.43031470777135516, + "grad_norm": 1.6061882049729523, + "learning_rate": 6.447808402117399e-06, + "loss": 0.6200252056121827, + "step": 2680 + }, + { + "epoch": 0.4319203596660244, + "grad_norm": 1.610665082618569, + "learning_rate": 6.4226379331551625e-06, + "loss": 0.6884698390960693, + "step": 2690 + }, + { + "epoch": 0.43352601156069365, + "grad_norm": 1.7081082215160799, + "learning_rate": 6.397428174258048e-06, + "loss": 0.6035865783691406, + "step": 2700 + }, + { + "epoch": 0.43513166345536286, + "grad_norm": 1.8692250975455653, + "learning_rate": 6.372179821660678e-06, + "loss": 0.6964015960693359, + "step": 2710 + }, + { + "epoch": 0.43673731535003213, + "grad_norm": 1.8172056429331103, + "learning_rate": 6.346893572663544e-06, + "loss": 0.6734973430633545, + "step": 2720 + }, + { + "epoch": 0.43834296724470134, + "grad_norm": 1.7475659871290619, + "learning_rate": 6.321570125613744e-06, + "loss": 0.6315503597259522, + "step": 2730 + }, + { + "epoch": 0.4399486191393706, + "grad_norm": 1.9819397747884144, + "learning_rate": 6.296210179885708e-06, + "loss": 0.7144417762756348, + "step": 2740 + }, + { + "epoch": 0.4415542710340398, + "grad_norm": 1.38229654099846, + "learning_rate": 6.270814435861864e-06, + "loss": 0.6535766124725342, + "step": 2750 + }, + { + "epoch": 0.44315992292870904, + "grad_norm": 1.5166113099535783, + "learning_rate": 6.245383594913312e-06, + "loss": 0.6655025482177734, + "step": 2760 + }, + { + "epoch": 0.4447655748233783, + "grad_norm": 1.775768796293097, + "learning_rate": 6.219918359380444e-06, + "loss": 0.6785330772399902, + "step": 2770 + }, + { + "epoch": 0.4463712267180475, + "grad_norm": 1.5738294130781245, + "learning_rate": 6.19441943255355e-06, + "loss": 0.6220744132995606, + "step": 2780 + }, + { + "epoch": 0.4479768786127168, + "grad_norm": 1.7091333372670319, + "learning_rate": 6.1688875186533955e-06, + "loss": 0.6746760368347168, + "step": 2790 + }, + { + "epoch": 0.449582530507386, + "grad_norm": 1.7540765777044425, + "learning_rate": 6.143323322811776e-06, + "loss": 0.6595097541809082, + "step": 2800 + }, + { + "epoch": 0.449582530507386, + "eval_loss": 0.6046592593193054, + "eval_runtime": 117.4341, + "eval_samples_per_second": 17.227, + "eval_steps_per_second": 4.309, + "eval_token_acc": 0.7942915635673181, + "step": 2800 + }, + { + "epoch": 0.4511881824020552, + "grad_norm": 1.716083495202826, + "learning_rate": 6.11772755105203e-06, + "loss": 0.6908525466918946, + "step": 2810 + }, + { + "epoch": 0.4527938342967245, + "grad_norm": 1.928873820576655, + "learning_rate": 6.092100910269556e-06, + "loss": 0.6462475776672363, + "step": 2820 + }, + { + "epoch": 0.4543994861913937, + "grad_norm": 1.7539886891121115, + "learning_rate": 6.06644410821228e-06, + "loss": 0.6449719905853272, + "step": 2830 + }, + { + "epoch": 0.45600513808606297, + "grad_norm": 1.75928350945977, + "learning_rate": 6.040757853461113e-06, + "loss": 0.6407927989959716, + "step": 2840 + }, + { + "epoch": 0.4576107899807322, + "grad_norm": 1.8255993162951358, + "learning_rate": 6.015042855410379e-06, + "loss": 0.6738231182098389, + "step": 2850 + }, + { + "epoch": 0.4592164418754014, + "grad_norm": 1.4721143485918229, + "learning_rate": 5.989299824248227e-06, + "loss": 0.6249940872192383, + "step": 2860 + }, + { + "epoch": 0.46082209377007066, + "grad_norm": 1.7825384178192991, + "learning_rate": 5.963529470937015e-06, + "loss": 0.6549758434295654, + "step": 2870 + }, + { + "epoch": 0.4624277456647399, + "grad_norm": 1.5080117937257649, + "learning_rate": 5.937732507193671e-06, + "loss": 0.6731220245361328, + "step": 2880 + }, + { + "epoch": 0.46403339755940914, + "grad_norm": 1.4123229770662427, + "learning_rate": 5.911909645470045e-06, + "loss": 0.6912211418151856, + "step": 2890 + }, + { + "epoch": 0.46563904945407836, + "grad_norm": 1.553691648545866, + "learning_rate": 5.886061598933228e-06, + "loss": 0.697138500213623, + "step": 2900 + }, + { + "epoch": 0.46724470134874757, + "grad_norm": 1.8969673644940426, + "learning_rate": 5.860189081445854e-06, + "loss": 0.6818105697631835, + "step": 2910 + }, + { + "epoch": 0.46885035324341684, + "grad_norm": 1.5267813484245596, + "learning_rate": 5.834292807546392e-06, + "loss": 0.6768715858459473, + "step": 2920 + }, + { + "epoch": 0.47045600513808605, + "grad_norm": 1.7034363371695085, + "learning_rate": 5.808373492429405e-06, + "loss": 0.6772209167480469, + "step": 2930 + }, + { + "epoch": 0.4720616570327553, + "grad_norm": 1.9475266449191355, + "learning_rate": 5.782431851925801e-06, + "loss": 0.6492327690124512, + "step": 2940 + }, + { + "epoch": 0.47366730892742454, + "grad_norm": 1.80685354115223, + "learning_rate": 5.75646860248306e-06, + "loss": 0.6602900505065918, + "step": 2950 + }, + { + "epoch": 0.47527296082209375, + "grad_norm": 1.5940654124212363, + "learning_rate": 5.730484461145455e-06, + "loss": 0.6496998786926269, + "step": 2960 + }, + { + "epoch": 0.476878612716763, + "grad_norm": 1.7231523708453245, + "learning_rate": 5.704480145534243e-06, + "loss": 0.634641456604004, + "step": 2970 + }, + { + "epoch": 0.47848426461143223, + "grad_norm": 1.6146352266889579, + "learning_rate": 5.678456373827843e-06, + "loss": 0.6208431720733643, + "step": 2980 + }, + { + "epoch": 0.4800899165061015, + "grad_norm": 1.6144539162269365, + "learning_rate": 5.652413864742016e-06, + "loss": 0.6912257194519043, + "step": 2990 + }, + { + "epoch": 0.4816955684007707, + "grad_norm": 1.7420176395616938, + "learning_rate": 5.626353337509994e-06, + "loss": 0.6629442691802978, + "step": 3000 + }, + { + "epoch": 0.4816955684007707, + "eval_loss": 0.6017531156539917, + "eval_runtime": 115.8046, + "eval_samples_per_second": 17.469, + "eval_steps_per_second": 4.369, + "eval_token_acc": 0.7953243645937789, + "step": 3000 + }, + { + "epoch": 0.4833012202954399, + "grad_norm": 1.5529601886653754, + "learning_rate": 5.600275511862636e-06, + "loss": 0.6236114501953125, + "step": 3010 + }, + { + "epoch": 0.4849068721901092, + "grad_norm": 1.7395798527530288, + "learning_rate": 5.574181108008539e-06, + "loss": 0.6623884201049804, + "step": 3020 + }, + { + "epoch": 0.4865125240847784, + "grad_norm": 1.5588683689028933, + "learning_rate": 5.548070846614153e-06, + "loss": 0.6456201076507568, + "step": 3030 + }, + { + "epoch": 0.4881181759794477, + "grad_norm": 1.6619854159322849, + "learning_rate": 5.521945448783874e-06, + "loss": 0.649496603012085, + "step": 3040 + }, + { + "epoch": 0.4897238278741169, + "grad_norm": 1.5892807391385169, + "learning_rate": 5.495805636040135e-06, + "loss": 0.6367308616638183, + "step": 3050 + }, + { + "epoch": 0.4913294797687861, + "grad_norm": 1.3345792855489715, + "learning_rate": 5.469652130303471e-06, + "loss": 0.6645633697509765, + "step": 3060 + }, + { + "epoch": 0.4929351316634554, + "grad_norm": 1.6214025907079206, + "learning_rate": 5.443485653872589e-06, + "loss": 0.6398410797119141, + "step": 3070 + }, + { + "epoch": 0.4945407835581246, + "grad_norm": 1.8451717722456522, + "learning_rate": 5.417306929404413e-06, + "loss": 0.7045126914978027, + "step": 3080 + }, + { + "epoch": 0.49614643545279385, + "grad_norm": 1.7162335450102724, + "learning_rate": 5.391116679894131e-06, + "loss": 0.6747453689575196, + "step": 3090 + }, + { + "epoch": 0.49775208734746307, + "grad_norm": 1.721924213910723, + "learning_rate": 5.364915628655227e-06, + "loss": 0.6267594337463379, + "step": 3100 + }, + { + "epoch": 0.4993577392421323, + "grad_norm": 1.8376897417896412, + "learning_rate": 5.3387044992995e-06, + "loss": 0.6397854804992675, + "step": 3110 + }, + { + "epoch": 0.5009633911368016, + "grad_norm": 1.869292144863384, + "learning_rate": 5.312484015717087e-06, + "loss": 0.6486954689025879, + "step": 3120 + }, + { + "epoch": 0.5025690430314708, + "grad_norm": 1.8544608956904656, + "learning_rate": 5.286254902056462e-06, + "loss": 0.689535665512085, + "step": 3130 + }, + { + "epoch": 0.50417469492614, + "grad_norm": 1.524780140770319, + "learning_rate": 5.2600178827044476e-06, + "loss": 0.5976661682128906, + "step": 3140 + }, + { + "epoch": 0.5057803468208093, + "grad_norm": 1.8393482466508677, + "learning_rate": 5.233773682266198e-06, + "loss": 0.652738380432129, + "step": 3150 + }, + { + "epoch": 0.5073859987154785, + "grad_norm": 1.680461273244531, + "learning_rate": 5.2075230255451924e-06, + "loss": 0.6367105484008789, + "step": 3160 + }, + { + "epoch": 0.5089916506101477, + "grad_norm": 1.4007542325300602, + "learning_rate": 5.181266637523225e-06, + "loss": 0.6549857139587403, + "step": 3170 + }, + { + "epoch": 0.5105973025048169, + "grad_norm": 1.7448220295886547, + "learning_rate": 5.155005243340364e-06, + "loss": 0.6184686183929443, + "step": 3180 + }, + { + "epoch": 0.5122029543994862, + "grad_norm": 1.66049201084311, + "learning_rate": 5.1287395682749444e-06, + "loss": 0.6485123634338379, + "step": 3190 + }, + { + "epoch": 0.5138086062941555, + "grad_norm": 1.521120678369957, + "learning_rate": 5.102470337723524e-06, + "loss": 0.6847625732421875, + "step": 3200 + }, + { + "epoch": 0.5138086062941555, + "eval_loss": 0.6003468036651611, + "eval_runtime": 117.3388, + "eval_samples_per_second": 17.241, + "eval_steps_per_second": 4.312, + "eval_token_acc": 0.7958849803367365, + "step": 3200 + }, + { + "epoch": 0.5154142581888247, + "grad_norm": 1.7475620617915268, + "learning_rate": 5.0761982771808595e-06, + "loss": 0.627868938446045, + "step": 3210 + }, + { + "epoch": 0.5170199100834939, + "grad_norm": 1.6492543156794781, + "learning_rate": 5.049924112219859e-06, + "loss": 0.6768051147460937, + "step": 3220 + }, + { + "epoch": 0.5186255619781631, + "grad_norm": 1.4702868123981876, + "learning_rate": 5.023648568471559e-06, + "loss": 0.6329309463500976, + "step": 3230 + }, + { + "epoch": 0.5202312138728323, + "grad_norm": 1.4631063429180151, + "learning_rate": 4.997372371605066e-06, + "loss": 0.6819337844848633, + "step": 3240 + }, + { + "epoch": 0.5218368657675017, + "grad_norm": 1.7194677170725678, + "learning_rate": 4.971096247307528e-06, + "loss": 0.6357659339904785, + "step": 3250 + }, + { + "epoch": 0.5234425176621709, + "grad_norm": 1.5391644286812827, + "learning_rate": 4.944820921264089e-06, + "loss": 0.6160742282867432, + "step": 3260 + }, + { + "epoch": 0.5250481695568401, + "grad_norm": 2.0292281910388414, + "learning_rate": 4.918547119137846e-06, + "loss": 0.6846878051757812, + "step": 3270 + }, + { + "epoch": 0.5266538214515093, + "grad_norm": 1.610879668884779, + "learning_rate": 4.89227556654981e-06, + "loss": 0.6450191020965577, + "step": 3280 + }, + { + "epoch": 0.5282594733461785, + "grad_norm": 1.339824263555254, + "learning_rate": 4.866006989058862e-06, + "loss": 0.6322847843170166, + "step": 3290 + }, + { + "epoch": 0.5298651252408478, + "grad_norm": 1.6543124351661145, + "learning_rate": 4.839742112141725e-06, + "loss": 0.6306018829345703, + "step": 3300 + }, + { + "epoch": 0.531470777135517, + "grad_norm": 1.660724826354808, + "learning_rate": 4.813481661172912e-06, + "loss": 0.610607099533081, + "step": 3310 + }, + { + "epoch": 0.5330764290301863, + "grad_norm": 1.6235734157752637, + "learning_rate": 4.787226361404706e-06, + "loss": 0.6995952606201172, + "step": 3320 + }, + { + "epoch": 0.5346820809248555, + "grad_norm": 1.868542231230083, + "learning_rate": 4.760976937947128e-06, + "loss": 0.6855478286743164, + "step": 3330 + }, + { + "epoch": 0.5362877328195247, + "grad_norm": 1.8331443076088598, + "learning_rate": 4.7347341157479055e-06, + "loss": 0.6339561462402343, + "step": 3340 + }, + { + "epoch": 0.537893384714194, + "grad_norm": 2.047103683230832, + "learning_rate": 4.708498619572455e-06, + "loss": 0.6341542720794677, + "step": 3350 + }, + { + "epoch": 0.5394990366088632, + "grad_norm": 1.6856551740545285, + "learning_rate": 4.682271173983865e-06, + "loss": 0.627877140045166, + "step": 3360 + }, + { + "epoch": 0.5411046885035324, + "grad_norm": 1.7444781147322475, + "learning_rate": 4.6560525033228885e-06, + "loss": 0.6696052551269531, + "step": 3370 + }, + { + "epoch": 0.5427103403982017, + "grad_norm": 1.818336504765959, + "learning_rate": 4.629843331687935e-06, + "loss": 0.6628055095672607, + "step": 3380 + }, + { + "epoch": 0.5443159922928709, + "grad_norm": 1.7541018451283887, + "learning_rate": 4.603644382915069e-06, + "loss": 0.6368642807006836, + "step": 3390 + }, + { + "epoch": 0.5459216441875402, + "grad_norm": 1.5154218896792584, + "learning_rate": 4.577456380558028e-06, + "loss": 0.6845203399658203, + "step": 3400 + }, + { + "epoch": 0.5459216441875402, + "eval_loss": 0.5972935557365417, + "eval_runtime": 115.9781, + "eval_samples_per_second": 17.443, + "eval_steps_per_second": 4.363, + "eval_token_acc": 0.7971763987446212, + "step": 3400 + }, + { + "epoch": 0.5475272960822094, + "grad_norm": 1.4551618658916237, + "learning_rate": 4.551280047868233e-06, + "loss": 0.639791202545166, + "step": 3410 + }, + { + "epoch": 0.5491329479768786, + "grad_norm": 1.892484597161116, + "learning_rate": 4.525116107774815e-06, + "loss": 0.6300538063049317, + "step": 3420 + }, + { + "epoch": 0.5507385998715478, + "grad_norm": 1.7651880235207764, + "learning_rate": 4.498965282864654e-06, + "loss": 0.7026824474334716, + "step": 3430 + }, + { + "epoch": 0.552344251766217, + "grad_norm": 1.6140223643155915, + "learning_rate": 4.472828295362417e-06, + "loss": 0.6442500114440918, + "step": 3440 + }, + { + "epoch": 0.5539499036608864, + "grad_norm": 1.7964942234618715, + "learning_rate": 4.446705867110613e-06, + "loss": 0.6450382232666015, + "step": 3450 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 1.5966192338898915, + "learning_rate": 4.420598719549661e-06, + "loss": 0.6269045829772949, + "step": 3460 + }, + { + "epoch": 0.5571612074502248, + "grad_norm": 1.6393500389444646, + "learning_rate": 4.39450757369796e-06, + "loss": 0.6151111602783204, + "step": 3470 + }, + { + "epoch": 0.558766859344894, + "grad_norm": 1.6460466953020223, + "learning_rate": 4.368433150131983e-06, + "loss": 0.6323574542999267, + "step": 3480 + }, + { + "epoch": 0.5603725112395632, + "grad_norm": 1.6778931367902616, + "learning_rate": 4.342376168966368e-06, + "loss": 0.6251521110534668, + "step": 3490 + }, + { + "epoch": 0.5619781631342325, + "grad_norm": 1.5349958675516662, + "learning_rate": 4.316337349834041e-06, + "loss": 0.622771167755127, + "step": 3500 + }, + { + "epoch": 0.5635838150289018, + "grad_norm": 1.7416286846749827, + "learning_rate": 4.290317411866329e-06, + "loss": 0.6873036384582519, + "step": 3510 + }, + { + "epoch": 0.565189466923571, + "grad_norm": 1.5581465818937876, + "learning_rate": 4.264317073673108e-06, + "loss": 0.5988898754119873, + "step": 3520 + }, + { + "epoch": 0.5667951188182402, + "grad_norm": 1.6185414612296933, + "learning_rate": 4.238337053322954e-06, + "loss": 0.624824333190918, + "step": 3530 + }, + { + "epoch": 0.5684007707129094, + "grad_norm": 1.9065074174136774, + "learning_rate": 4.212378068323312e-06, + "loss": 0.673931360244751, + "step": 3540 + }, + { + "epoch": 0.5700064226075787, + "grad_norm": 1.656265815281467, + "learning_rate": 4.186440835600677e-06, + "loss": 0.6439894676208496, + "step": 3550 + }, + { + "epoch": 0.5716120745022479, + "grad_norm": 1.8776786172892783, + "learning_rate": 4.1605260714808e-06, + "loss": 0.6581153869628906, + "step": 3560 + }, + { + "epoch": 0.5732177263969171, + "grad_norm": 1.703747054653242, + "learning_rate": 4.134634491668903e-06, + "loss": 0.61540846824646, + "step": 3570 + }, + { + "epoch": 0.5748233782915864, + "grad_norm": 1.5507582889339149, + "learning_rate": 4.108766811229906e-06, + "loss": 0.640535831451416, + "step": 3580 + }, + { + "epoch": 0.5764290301862556, + "grad_norm": 1.618375014888649, + "learning_rate": 4.0829237445686895e-06, + "loss": 0.6489696502685547, + "step": 3590 + }, + { + "epoch": 0.5780346820809249, + "grad_norm": 1.752024337684377, + "learning_rate": 4.057106005410356e-06, + "loss": 0.6690279960632324, + "step": 3600 + }, + { + "epoch": 0.5780346820809249, + "eval_loss": 0.5927916765213013, + "eval_runtime": 115.8054, + "eval_samples_per_second": 17.469, + "eval_steps_per_second": 4.369, + "eval_token_acc": 0.7976844567616765, + "step": 3600 + }, + { + "epoch": 0.5796403339755941, + "grad_norm": 1.5341837355565895, + "learning_rate": 4.0313143067805255e-06, + "loss": 0.6698870658874512, + "step": 3610 + }, + { + "epoch": 0.5812459858702633, + "grad_norm": 1.6467496057484918, + "learning_rate": 4.005549360985633e-06, + "loss": 0.5921213150024414, + "step": 3620 + }, + { + "epoch": 0.5828516377649325, + "grad_norm": 1.6053152728267086, + "learning_rate": 3.979811879593269e-06, + "loss": 0.6558093070983887, + "step": 3630 + }, + { + "epoch": 0.5844572896596018, + "grad_norm": 1.9032471949878327, + "learning_rate": 3.954102573412517e-06, + "loss": 0.6461283683776855, + "step": 3640 + }, + { + "epoch": 0.5860629415542711, + "grad_norm": 1.689193286628274, + "learning_rate": 3.9284221524743285e-06, + "loss": 0.6304150581359863, + "step": 3650 + }, + { + "epoch": 0.5876685934489403, + "grad_norm": 1.6774991480461452, + "learning_rate": 3.902771326011914e-06, + "loss": 0.6262317657470703, + "step": 3660 + }, + { + "epoch": 0.5892742453436095, + "grad_norm": 1.8474226985791815, + "learning_rate": 3.877150802441151e-06, + "loss": 0.6774571418762207, + "step": 3670 + }, + { + "epoch": 0.5908798972382787, + "grad_norm": 1.6063158015617602, + "learning_rate": 3.851561289341023e-06, + "loss": 0.6113105297088623, + "step": 3680 + }, + { + "epoch": 0.5924855491329479, + "grad_norm": 1.6990382513407551, + "learning_rate": 3.8260034934340774e-06, + "loss": 0.632629108428955, + "step": 3690 + }, + { + "epoch": 0.5940912010276173, + "grad_norm": 1.7284057501817178, + "learning_rate": 3.800478120566906e-06, + "loss": 0.6128873348236084, + "step": 3700 + }, + { + "epoch": 0.5956968529222865, + "grad_norm": 1.7913233473229204, + "learning_rate": 3.7749858756906516e-06, + "loss": 0.6623647689819336, + "step": 3710 + }, + { + "epoch": 0.5973025048169557, + "grad_norm": 1.8150662300786826, + "learning_rate": 3.749527462841539e-06, + "loss": 0.6639890670776367, + "step": 3720 + }, + { + "epoch": 0.5989081567116249, + "grad_norm": 1.5307570049241765, + "learning_rate": 3.724103585121436e-06, + "loss": 0.6226850032806397, + "step": 3730 + }, + { + "epoch": 0.6005138086062941, + "grad_norm": 1.6481951251937415, + "learning_rate": 3.698714944678424e-06, + "loss": 0.6433711528778077, + "step": 3740 + }, + { + "epoch": 0.6021194605009634, + "grad_norm": 1.8311326245894355, + "learning_rate": 3.6733622426874184e-06, + "loss": 0.6615542411804199, + "step": 3750 + }, + { + "epoch": 0.6037251123956326, + "grad_norm": 1.695513446479562, + "learning_rate": 3.648046179330796e-06, + "loss": 0.6288583755493165, + "step": 3760 + }, + { + "epoch": 0.6053307642903019, + "grad_norm": 1.8554703021185914, + "learning_rate": 3.62276745377906e-06, + "loss": 0.668128490447998, + "step": 3770 + }, + { + "epoch": 0.6069364161849711, + "grad_norm": 1.8414598334221723, + "learning_rate": 3.597526764171532e-06, + "loss": 0.6891838073730469, + "step": 3780 + }, + { + "epoch": 0.6085420680796403, + "grad_norm": 1.544742622918396, + "learning_rate": 3.5723248075970684e-06, + "loss": 0.6486976146697998, + "step": 3790 + }, + { + "epoch": 0.6101477199743096, + "grad_norm": 1.5027378360941297, + "learning_rate": 3.547162280074813e-06, + "loss": 0.6122174263000488, + "step": 3800 + }, + { + "epoch": 0.6101477199743096, + "eval_loss": 0.5898537635803223, + "eval_runtime": 115.7711, + "eval_samples_per_second": 17.474, + "eval_steps_per_second": 4.371, + "eval_token_acc": 0.7986213190821252, + "step": 3800 + }, + { + "epoch": 0.6117533718689788, + "grad_norm": 1.56493074871109, + "learning_rate": 3.5220398765349662e-06, + "loss": 0.592191743850708, + "step": 3810 + }, + { + "epoch": 0.613359023763648, + "grad_norm": 1.5559314443165084, + "learning_rate": 3.4969582907996015e-06, + "loss": 0.7172041893005371, + "step": 3820 + }, + { + "epoch": 0.6149646756583173, + "grad_norm": 1.881786450582419, + "learning_rate": 3.471918215563499e-06, + "loss": 0.6604925155639648, + "step": 3830 + }, + { + "epoch": 0.6165703275529865, + "grad_norm": 1.6686394422851605, + "learning_rate": 3.4469203423750152e-06, + "loss": 0.6463912487030029, + "step": 3840 + }, + { + "epoch": 0.6181759794476558, + "grad_norm": 1.7643375635456229, + "learning_rate": 3.421965361616985e-06, + "loss": 0.6369779109954834, + "step": 3850 + }, + { + "epoch": 0.619781631342325, + "grad_norm": 1.723114857381474, + "learning_rate": 3.3970539624876565e-06, + "loss": 0.7152776718139648, + "step": 3860 + }, + { + "epoch": 0.6213872832369942, + "grad_norm": 1.6636836288686383, + "learning_rate": 3.372186832981652e-06, + "loss": 0.6432122230529785, + "step": 3870 + }, + { + "epoch": 0.6229929351316634, + "grad_norm": 1.8013891223015748, + "learning_rate": 3.3473646598709724e-06, + "loss": 0.6690933227539062, + "step": 3880 + }, + { + "epoch": 0.6245985870263326, + "grad_norm": 1.832975956776872, + "learning_rate": 3.322588128686027e-06, + "loss": 0.573091459274292, + "step": 3890 + }, + { + "epoch": 0.626204238921002, + "grad_norm": 1.8006171372641149, + "learning_rate": 3.297857923696702e-06, + "loss": 0.641511344909668, + "step": 3900 + }, + { + "epoch": 0.6278098908156712, + "grad_norm": 1.6491853355530541, + "learning_rate": 3.273174727893463e-06, + "loss": 0.6224217414855957, + "step": 3910 + }, + { + "epoch": 0.6294155427103404, + "grad_norm": 1.7497490967216878, + "learning_rate": 3.248539222968489e-06, + "loss": 0.6409141540527343, + "step": 3920 + }, + { + "epoch": 0.6310211946050096, + "grad_norm": 1.7397516831302413, + "learning_rate": 3.223952089296854e-06, + "loss": 0.6265341758728027, + "step": 3930 + }, + { + "epoch": 0.6326268464996788, + "grad_norm": 1.5441779493196535, + "learning_rate": 3.199414005917726e-06, + "loss": 0.6394174098968506, + "step": 3940 + }, + { + "epoch": 0.6342324983943481, + "grad_norm": 1.649609687364375, + "learning_rate": 3.1749256505156203e-06, + "loss": 0.6349728107452393, + "step": 3950 + }, + { + "epoch": 0.6358381502890174, + "grad_norm": 1.782563240036328, + "learning_rate": 3.150487699401681e-06, + "loss": 0.6500693798065186, + "step": 3960 + }, + { + "epoch": 0.6374438021836866, + "grad_norm": 1.8528260207135643, + "learning_rate": 3.1261008274950045e-06, + "loss": 0.630257511138916, + "step": 3970 + }, + { + "epoch": 0.6390494540783558, + "grad_norm": 1.936677866102448, + "learning_rate": 3.1017657083039974e-06, + "loss": 0.6424071788787842, + "step": 3980 + }, + { + "epoch": 0.640655105973025, + "grad_norm": 1.568379134898161, + "learning_rate": 3.0774830139077816e-06, + "loss": 0.6053402423858643, + "step": 3990 + }, + { + "epoch": 0.6422607578676943, + "grad_norm": 1.514777836052795, + "learning_rate": 3.0532534149376225e-06, + "loss": 0.5991374492645264, + "step": 4000 + }, + { + "epoch": 0.6422607578676943, + "eval_loss": 0.5840339064598083, + "eval_runtime": 116.031, + "eval_samples_per_second": 17.435, + "eval_steps_per_second": 4.361, + "eval_token_acc": 0.7992353268006026, + "step": 4000 + }, + { + "epoch": 0.6438664097623635, + "grad_norm": 1.7950486729629296, + "learning_rate": 3.0290775805584182e-06, + "loss": 0.6125258445739746, + "step": 4010 + }, + { + "epoch": 0.6454720616570327, + "grad_norm": 1.6708646123073856, + "learning_rate": 3.0049561784502125e-06, + "loss": 0.6363659858703613, + "step": 4020 + }, + { + "epoch": 0.647077713551702, + "grad_norm": 1.692649611123861, + "learning_rate": 2.980889874789758e-06, + "loss": 0.6517571926116943, + "step": 4030 + }, + { + "epoch": 0.6486833654463712, + "grad_norm": 1.3401264023308128, + "learning_rate": 2.956879334232117e-06, + "loss": 0.6088146686553955, + "step": 4040 + }, + { + "epoch": 0.6502890173410405, + "grad_norm": 1.5769633797907998, + "learning_rate": 2.9329252198923026e-06, + "loss": 0.6375810623168945, + "step": 4050 + }, + { + "epoch": 0.6518946692357097, + "grad_norm": 1.7284597516150746, + "learning_rate": 2.909028193326974e-06, + "loss": 0.6683880805969238, + "step": 4060 + }, + { + "epoch": 0.6535003211303789, + "grad_norm": 1.856703537001769, + "learning_rate": 2.8851889145161515e-06, + "loss": 0.6554866313934327, + "step": 4070 + }, + { + "epoch": 0.6551059730250481, + "grad_norm": 1.953161442150085, + "learning_rate": 2.861408041845002e-06, + "loss": 0.6188833236694335, + "step": 4080 + }, + { + "epoch": 0.6567116249197174, + "grad_norm": 1.7195675953857033, + "learning_rate": 2.8376862320856524e-06, + "loss": 0.6229765892028809, + "step": 4090 + }, + { + "epoch": 0.6583172768143867, + "grad_norm": 1.8260724058291402, + "learning_rate": 2.814024140379048e-06, + "loss": 0.6316720008850097, + "step": 4100 + }, + { + "epoch": 0.6599229287090559, + "grad_norm": 1.9499916194335019, + "learning_rate": 2.7904224202168608e-06, + "loss": 0.5982970237731934, + "step": 4110 + }, + { + "epoch": 0.6615285806037251, + "grad_norm": 1.8038413382260061, + "learning_rate": 2.766881723423441e-06, + "loss": 0.6522846221923828, + "step": 4120 + }, + { + "epoch": 0.6631342324983943, + "grad_norm": 1.542415640870293, + "learning_rate": 2.7434027001378194e-06, + "loss": 0.6323680877685547, + "step": 4130 + }, + { + "epoch": 0.6647398843930635, + "grad_norm": 1.5534418727849721, + "learning_rate": 2.719985998795747e-06, + "loss": 0.6778179168701172, + "step": 4140 + }, + { + "epoch": 0.6663455362877329, + "grad_norm": 1.5712742360039995, + "learning_rate": 2.696632266111784e-06, + "loss": 0.6367866516113281, + "step": 4150 + }, + { + "epoch": 0.6679511881824021, + "grad_norm": 1.833593699902694, + "learning_rate": 2.67334214706145e-06, + "loss": 0.6607365608215332, + "step": 4160 + }, + { + "epoch": 0.6695568400770713, + "grad_norm": 1.644689383370669, + "learning_rate": 2.6501162848634023e-06, + "loss": 0.6528053760528565, + "step": 4170 + }, + { + "epoch": 0.6711624919717405, + "grad_norm": 1.8086124772278036, + "learning_rate": 2.6269553209616705e-06, + "loss": 0.6112563133239746, + "step": 4180 + }, + { + "epoch": 0.6727681438664097, + "grad_norm": 1.9128089462595501, + "learning_rate": 2.603859895007953e-06, + "loss": 0.6744743347167969, + "step": 4190 + }, + { + "epoch": 0.674373795761079, + "grad_norm": 1.503328791218411, + "learning_rate": 2.5808306448439363e-06, + "loss": 0.6157184600830078, + "step": 4200 + }, + { + "epoch": 0.674373795761079, + "eval_loss": 0.5820798873901367, + "eval_runtime": 115.5668, + "eval_samples_per_second": 17.505, + "eval_steps_per_second": 4.378, + "eval_token_acc": 0.8002397633400685, + "step": 4200 + }, + { + "epoch": 0.6759794476557482, + "grad_norm": 2.045147193567602, + "learning_rate": 2.557868206483689e-06, + "loss": 0.6391608238220214, + "step": 4210 + }, + { + "epoch": 0.6775850995504175, + "grad_norm": 1.7147802927742894, + "learning_rate": 2.5349732140960924e-06, + "loss": 0.6371169567108155, + "step": 4220 + }, + { + "epoch": 0.6791907514450867, + "grad_norm": 1.685988655934266, + "learning_rate": 2.5121462999873304e-06, + "loss": 0.5960301876068115, + "step": 4230 + }, + { + "epoch": 0.680796403339756, + "grad_norm": 1.6786670117397713, + "learning_rate": 2.48938809458342e-06, + "loss": 0.6312998294830322, + "step": 4240 + }, + { + "epoch": 0.6824020552344252, + "grad_norm": 1.4873973912798752, + "learning_rate": 2.466699226412807e-06, + "loss": 0.6012437343597412, + "step": 4250 + }, + { + "epoch": 0.6840077071290944, + "grad_norm": 1.7189122820194944, + "learning_rate": 2.4440803220890054e-06, + "loss": 0.6181604385375976, + "step": 4260 + }, + { + "epoch": 0.6856133590237636, + "grad_norm": 1.7992978742198917, + "learning_rate": 2.4215320062932884e-06, + "loss": 0.616178321838379, + "step": 4270 + }, + { + "epoch": 0.6872190109184328, + "grad_norm": 1.633995574685268, + "learning_rate": 2.399054901757442e-06, + "loss": 0.6509074211120606, + "step": 4280 + }, + { + "epoch": 0.6888246628131022, + "grad_norm": 1.8247436739266203, + "learning_rate": 2.3766496292465626e-06, + "loss": 0.6111636161804199, + "step": 4290 + }, + { + "epoch": 0.6904303147077714, + "grad_norm": 2.010698175443969, + "learning_rate": 2.3543168075419128e-06, + "loss": 0.6828377723693848, + "step": 4300 + }, + { + "epoch": 0.6920359666024406, + "grad_norm": 1.5603806270643172, + "learning_rate": 2.3320570534238333e-06, + "loss": 0.6230679512023926, + "step": 4310 + }, + { + "epoch": 0.6936416184971098, + "grad_norm": 1.7656937668495685, + "learning_rate": 2.3098709816547126e-06, + "loss": 0.6073711395263672, + "step": 4320 + }, + { + "epoch": 0.695247270391779, + "grad_norm": 1.5675790546975539, + "learning_rate": 2.2877592049620013e-06, + "loss": 0.6132484912872315, + "step": 4330 + }, + { + "epoch": 0.6968529222864484, + "grad_norm": 1.8840928192176054, + "learning_rate": 2.2657223340212937e-06, + "loss": 0.6499895095825196, + "step": 4340 + }, + { + "epoch": 0.6984585741811176, + "grad_norm": 1.485949916458336, + "learning_rate": 2.243760977439463e-06, + "loss": 0.615330696105957, + "step": 4350 + }, + { + "epoch": 0.7000642260757868, + "grad_norm": 1.7164083590388903, + "learning_rate": 2.2218757417378524e-06, + "loss": 0.6341984748840332, + "step": 4360 + }, + { + "epoch": 0.701669877970456, + "grad_norm": 1.4624278832608588, + "learning_rate": 2.2000672313355243e-06, + "loss": 0.5987278938293457, + "step": 4370 + }, + { + "epoch": 0.7032755298651252, + "grad_norm": 1.7173759962972308, + "learning_rate": 2.178336048532567e-06, + "loss": 0.601482629776001, + "step": 4380 + }, + { + "epoch": 0.7048811817597945, + "grad_norm": 1.7808810749639428, + "learning_rate": 2.1566827934934625e-06, + "loss": 0.667569351196289, + "step": 4390 + }, + { + "epoch": 0.7064868336544637, + "grad_norm": 1.4738305673809178, + "learning_rate": 2.1351080642305087e-06, + "loss": 0.6533387660980224, + "step": 4400 + }, + { + "epoch": 0.7064868336544637, + "eval_loss": 0.580632746219635, + "eval_runtime": 117.1398, + "eval_samples_per_second": 17.27, + "eval_steps_per_second": 4.32, + "eval_token_acc": 0.800744484358654, + "step": 4400 + }, + { + "epoch": 0.708092485549133, + "grad_norm": 1.6325615237394804, + "learning_rate": 2.1136124565873067e-06, + "loss": 0.5981454372406005, + "step": 4410 + }, + { + "epoch": 0.7096981374438022, + "grad_norm": 1.4296787875753854, + "learning_rate": 2.092196564222301e-06, + "loss": 0.6405043601989746, + "step": 4420 + }, + { + "epoch": 0.7113037893384714, + "grad_norm": 1.949785258946256, + "learning_rate": 2.070860978592389e-06, + "loss": 0.6880702972412109, + "step": 4430 + }, + { + "epoch": 0.7129094412331407, + "grad_norm": 1.7526041237317675, + "learning_rate": 2.04960628893658e-06, + "loss": 0.6139644622802735, + "step": 4440 + }, + { + "epoch": 0.7145150931278099, + "grad_norm": 1.5476358574696476, + "learning_rate": 2.0284330822597328e-06, + "loss": 0.6332278251647949, + "step": 4450 + }, + { + "epoch": 0.7161207450224791, + "grad_norm": 1.8500596491674761, + "learning_rate": 2.0073419433163287e-06, + "loss": 0.6281499862670898, + "step": 4460 + }, + { + "epoch": 0.7177263969171483, + "grad_norm": 1.7651733298943222, + "learning_rate": 1.9863334545943346e-06, + "loss": 0.6181722640991211, + "step": 4470 + }, + { + "epoch": 0.7193320488118176, + "grad_norm": 1.6049924741534642, + "learning_rate": 1.96540819629911e-06, + "loss": 0.6430582523345947, + "step": 4480 + }, + { + "epoch": 0.7209377007064869, + "grad_norm": 1.4421025800768663, + "learning_rate": 1.944566746337384e-06, + "loss": 0.6036171913146973, + "step": 4490 + }, + { + "epoch": 0.7225433526011561, + "grad_norm": 1.6285363292438688, + "learning_rate": 1.9238096803012977e-06, + "loss": 0.6362571716308594, + "step": 4500 + }, + { + "epoch": 0.7241490044958253, + "grad_norm": 1.7068600729705337, + "learning_rate": 1.9031375714525024e-06, + "loss": 0.6020670890808105, + "step": 4510 + }, + { + "epoch": 0.7257546563904945, + "grad_norm": 1.8694371561958198, + "learning_rate": 1.8825509907063328e-06, + "loss": 0.6477363109588623, + "step": 4520 + }, + { + "epoch": 0.7273603082851637, + "grad_norm": 1.5483620034723313, + "learning_rate": 1.862050506616036e-06, + "loss": 0.634291410446167, + "step": 4530 + }, + { + "epoch": 0.7289659601798331, + "grad_norm": 1.6864092934529722, + "learning_rate": 1.841636685357071e-06, + "loss": 0.6551846027374267, + "step": 4540 + }, + { + "epoch": 0.7305716120745023, + "grad_norm": 1.6916900725584771, + "learning_rate": 1.8213100907114723e-06, + "loss": 0.6069769859313965, + "step": 4550 + }, + { + "epoch": 0.7321772639691715, + "grad_norm": 1.6735002405038526, + "learning_rate": 1.8010712840522787e-06, + "loss": 0.6614324569702148, + "step": 4560 + }, + { + "epoch": 0.7337829158638407, + "grad_norm": 1.8585454189612793, + "learning_rate": 1.7809208243280295e-06, + "loss": 0.6140747547149659, + "step": 4570 + }, + { + "epoch": 0.7353885677585099, + "grad_norm": 1.67101610321688, + "learning_rate": 1.7608592680473286e-06, + "loss": 0.596295166015625, + "step": 4580 + }, + { + "epoch": 0.7369942196531792, + "grad_norm": 1.6090351086530328, + "learning_rate": 1.740887169263477e-06, + "loss": 0.6292660236358643, + "step": 4590 + }, + { + "epoch": 0.7385998715478485, + "grad_norm": 1.6470392339720403, + "learning_rate": 1.7210050795591659e-06, + "loss": 0.6159173011779785, + "step": 4600 + }, + { + "epoch": 0.7385998715478485, + "eval_loss": 0.5773460865020752, + "eval_runtime": 115.7364, + "eval_samples_per_second": 17.479, + "eval_steps_per_second": 4.372, + "eval_token_acc": 0.8012725643665293, + "step": 4600 + }, + { + "epoch": 0.7402055234425177, + "grad_norm": 1.9918438283277915, + "learning_rate": 1.7012135480312453e-06, + "loss": 0.700563621520996, + "step": 4610 + }, + { + "epoch": 0.7418111753371869, + "grad_norm": 1.6743656898343373, + "learning_rate": 1.681513121275562e-06, + "loss": 0.5890089988708496, + "step": 4620 + }, + { + "epoch": 0.7434168272318561, + "grad_norm": 1.7197766351679413, + "learning_rate": 1.6619043433718618e-06, + "loss": 0.5891599655151367, + "step": 4630 + }, + { + "epoch": 0.7450224791265254, + "grad_norm": 1.8377522310891323, + "learning_rate": 1.6423877558687618e-06, + "loss": 0.6465985298156738, + "step": 4640 + }, + { + "epoch": 0.7466281310211946, + "grad_norm": 1.693859000264474, + "learning_rate": 1.6229638977687978e-06, + "loss": 0.6260784149169922, + "step": 4650 + }, + { + "epoch": 0.7482337829158638, + "grad_norm": 1.7224187559205801, + "learning_rate": 1.6036333055135345e-06, + "loss": 0.6414599418640137, + "step": 4660 + }, + { + "epoch": 0.7498394348105331, + "grad_norm": 1.470812358394573, + "learning_rate": 1.5843965129687534e-06, + "loss": 0.5949658870697021, + "step": 4670 + }, + { + "epoch": 0.7514450867052023, + "grad_norm": 1.9470524033349248, + "learning_rate": 1.5652540514097053e-06, + "loss": 0.6360251426696777, + "step": 4680 + }, + { + "epoch": 0.7530507385998716, + "grad_norm": 1.502544150605463, + "learning_rate": 1.5462064495064422e-06, + "loss": 0.642902946472168, + "step": 4690 + }, + { + "epoch": 0.7546563904945408, + "grad_norm": 1.8899266627864577, + "learning_rate": 1.5272542333092111e-06, + "loss": 0.6440675258636475, + "step": 4700 + }, + { + "epoch": 0.75626204238921, + "grad_norm": 1.8027721737162652, + "learning_rate": 1.5083979262339299e-06, + "loss": 0.6398180961608887, + "step": 4710 + }, + { + "epoch": 0.7578676942838792, + "grad_norm": 2.0384841764171733, + "learning_rate": 1.4896380490477336e-06, + "loss": 0.6641289710998535, + "step": 4720 + }, + { + "epoch": 0.7594733461785484, + "grad_norm": 1.6291499323229892, + "learning_rate": 1.4709751198545858e-06, + "loss": 0.6209500789642334, + "step": 4730 + }, + { + "epoch": 0.7610789980732178, + "grad_norm": 1.7406815962711, + "learning_rate": 1.4524096540809746e-06, + "loss": 0.6143218040466308, + "step": 4740 + }, + { + "epoch": 0.762684649967887, + "grad_norm": 1.6318467742073794, + "learning_rate": 1.4339421644616723e-06, + "loss": 0.602264404296875, + "step": 4750 + }, + { + "epoch": 0.7642903018625562, + "grad_norm": 1.6983566290312326, + "learning_rate": 1.415573161025584e-06, + "loss": 0.5744462013244629, + "step": 4760 + }, + { + "epoch": 0.7658959537572254, + "grad_norm": 1.5568550381005917, + "learning_rate": 1.3973031510816542e-06, + "loss": 0.6219449520111084, + "step": 4770 + }, + { + "epoch": 0.7675016056518946, + "grad_norm": 1.7262384743458887, + "learning_rate": 1.3791326392048593e-06, + "loss": 0.6433005809783936, + "step": 4780 + }, + { + "epoch": 0.769107257546564, + "grad_norm": 1.8571619626510951, + "learning_rate": 1.3610621272222713e-06, + "loss": 0.6154828548431397, + "step": 4790 + }, + { + "epoch": 0.7707129094412332, + "grad_norm": 1.5136725460512475, + "learning_rate": 1.3430921141991977e-06, + "loss": 0.6346235275268555, + "step": 4800 + }, + { + "epoch": 0.7707129094412332, + "eval_loss": 0.5772837996482849, + "eval_runtime": 117.978, + "eval_samples_per_second": 17.147, + "eval_steps_per_second": 4.289, + "eval_token_acc": 0.8020625987542984, + "step": 4800 + }, + { + "epoch": 0.7723185613359024, + "grad_norm": 1.941479206845148, + "learning_rate": 1.3252230964253998e-06, + "loss": 0.6047333717346192, + "step": 4810 + }, + { + "epoch": 0.7739242132305716, + "grad_norm": 1.4802656853080443, + "learning_rate": 1.3074555674013901e-06, + "loss": 0.6177777290344239, + "step": 4820 + }, + { + "epoch": 0.7755298651252408, + "grad_norm": 1.8409632389077035, + "learning_rate": 1.2897900178247945e-06, + "loss": 0.6657865524291993, + "step": 4830 + }, + { + "epoch": 0.7771355170199101, + "grad_norm": 1.7465471409023423, + "learning_rate": 1.2722269355768058e-06, + "loss": 0.6325933456420898, + "step": 4840 + }, + { + "epoch": 0.7787411689145793, + "grad_norm": 1.7357172898681474, + "learning_rate": 1.2547668057087097e-06, + "loss": 0.6073914527893066, + "step": 4850 + }, + { + "epoch": 0.7803468208092486, + "grad_norm": 1.5826493279336413, + "learning_rate": 1.237410110428487e-06, + "loss": 0.6457367420196534, + "step": 4860 + }, + { + "epoch": 0.7819524727039178, + "grad_norm": 2.169817614599634, + "learning_rate": 1.2201573290874963e-06, + "loss": 0.5977273941040039, + "step": 4870 + }, + { + "epoch": 0.783558124598587, + "grad_norm": 1.6865545638523116, + "learning_rate": 1.2030089381672384e-06, + "loss": 0.6518450260162354, + "step": 4880 + }, + { + "epoch": 0.7851637764932563, + "grad_norm": 1.8293621274972247, + "learning_rate": 1.1859654112661923e-06, + "loss": 0.6506802082061768, + "step": 4890 + }, + { + "epoch": 0.7867694283879255, + "grad_norm": 1.8105963381339787, + "learning_rate": 1.169027219086739e-06, + "loss": 0.6600035667419434, + "step": 4900 + }, + { + "epoch": 0.7883750802825947, + "grad_norm": 1.604620184322651, + "learning_rate": 1.1521948294221603e-06, + "loss": 0.6555306434631347, + "step": 4910 + }, + { + "epoch": 0.789980732177264, + "grad_norm": 1.7855453593256356, + "learning_rate": 1.1354687071437197e-06, + "loss": 0.5738696575164794, + "step": 4920 + }, + { + "epoch": 0.7915863840719332, + "grad_norm": 1.6455556925524137, + "learning_rate": 1.1188493141878248e-06, + "loss": 0.6280563354492188, + "step": 4930 + }, + { + "epoch": 0.7931920359666025, + "grad_norm": 1.5084264332429083, + "learning_rate": 1.1023371095432656e-06, + "loss": 0.6412508010864257, + "step": 4940 + }, + { + "epoch": 0.7947976878612717, + "grad_norm": 1.748454465592618, + "learning_rate": 1.085932549238547e-06, + "loss": 0.6463836669921875, + "step": 4950 + }, + { + "epoch": 0.7964033397559409, + "grad_norm": 1.5700493018887136, + "learning_rate": 1.0696360863292842e-06, + "loss": 0.6505722045898438, + "step": 4960 + }, + { + "epoch": 0.7980089916506101, + "grad_norm": 2.1173458296345244, + "learning_rate": 1.053448170885697e-06, + "loss": 0.5998007774353027, + "step": 4970 + }, + { + "epoch": 0.7996146435452793, + "grad_norm": 1.7193892335616814, + "learning_rate": 1.0373692499801763e-06, + "loss": 0.6597327709197998, + "step": 4980 + }, + { + "epoch": 0.8012202954399487, + "grad_norm": 1.5021327733245695, + "learning_rate": 1.021399767674941e-06, + "loss": 0.6283426761627198, + "step": 4990 + }, + { + "epoch": 0.8028259473346179, + "grad_norm": 2.050460593018747, + "learning_rate": 1.0055401650097685e-06, + "loss": 0.6920224189758301, + "step": 5000 + }, + { + "epoch": 0.8028259473346179, + "eval_loss": 0.5766663551330566, + "eval_runtime": 116.4968, + "eval_samples_per_second": 17.365, + "eval_steps_per_second": 4.343, + "eval_token_acc": 0.8022861776517876, + "step": 5000 + }, + { + "epoch": 0.8044315992292871, + "grad_norm": 1.5486550221902147, + "learning_rate": 9.89790879989821e-07, + "loss": 0.6464922428131104, + "step": 5010 + }, + { + "epoch": 0.8060372511239563, + "grad_norm": 1.6738208657333828, + "learning_rate": 9.741523475735414e-07, + "loss": 0.6773768424987793, + "step": 5020 + }, + { + "epoch": 0.8076429030186255, + "grad_norm": 1.600400825176277, + "learning_rate": 9.586249996606473e-07, + "loss": 0.6198241710662842, + "step": 5030 + }, + { + "epoch": 0.8092485549132948, + "grad_norm": 1.604271483427513, + "learning_rate": 9.432092650801994e-07, + "loss": 0.644795274734497, + "step": 5040 + }, + { + "epoch": 0.810854206807964, + "grad_norm": 1.7603640838795784, + "learning_rate": 9.279055695787582e-07, + "loss": 0.5961036682128906, + "step": 5050 + }, + { + "epoch": 0.8124598587026333, + "grad_norm": 1.6707735318483077, + "learning_rate": 9.127143358086277e-07, + "loss": 0.6223243713378906, + "step": 5060 + }, + { + "epoch": 0.8140655105973025, + "grad_norm": 1.4399237322867773, + "learning_rate": 8.976359833161796e-07, + "loss": 0.6160260200500488, + "step": 5070 + }, + { + "epoch": 0.8156711624919717, + "grad_norm": 1.5683323659854056, + "learning_rate": 8.826709285302737e-07, + "loss": 0.6315612316131591, + "step": 5080 + }, + { + "epoch": 0.817276814386641, + "grad_norm": 2.0831948488502423, + "learning_rate": 8.678195847507464e-07, + "loss": 0.6538731098175049, + "step": 5090 + }, + { + "epoch": 0.8188824662813102, + "grad_norm": 1.7432408963106332, + "learning_rate": 8.530823621370043e-07, + "loss": 0.6310447216033935, + "step": 5100 + }, + { + "epoch": 0.8204881181759794, + "grad_norm": 1.5823330528149908, + "learning_rate": 8.384596676966938e-07, + "loss": 0.5536775112152099, + "step": 5110 + }, + { + "epoch": 0.8220937700706487, + "grad_norm": 1.7106531605451956, + "learning_rate": 8.239519052744605e-07, + "loss": 0.6286486625671387, + "step": 5120 + }, + { + "epoch": 0.8236994219653179, + "grad_norm": 1.936155046132306, + "learning_rate": 8.095594755407971e-07, + "loss": 0.6777469635009765, + "step": 5130 + }, + { + "epoch": 0.8253050738599872, + "grad_norm": 1.3901083055492947, + "learning_rate": 7.952827759809756e-07, + "loss": 0.6293821811676026, + "step": 5140 + }, + { + "epoch": 0.8269107257546564, + "grad_norm": 1.718191378615406, + "learning_rate": 7.811222008840719e-07, + "loss": 0.6516339302062988, + "step": 5150 + }, + { + "epoch": 0.8285163776493256, + "grad_norm": 1.902823159131978, + "learning_rate": 7.670781413320766e-07, + "loss": 0.5841949462890625, + "step": 5160 + }, + { + "epoch": 0.8301220295439948, + "grad_norm": 1.6396798192512863, + "learning_rate": 7.531509851890911e-07, + "loss": 0.6197714328765869, + "step": 5170 + }, + { + "epoch": 0.831727681438664, + "grad_norm": 1.753661211335331, + "learning_rate": 7.393411170906201e-07, + "loss": 0.6171721935272216, + "step": 5180 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 1.5127497599851492, + "learning_rate": 7.256489184329452e-07, + "loss": 0.604136323928833, + "step": 5190 + }, + { + "epoch": 0.8349389852280026, + "grad_norm": 1.4904625841129133, + "learning_rate": 7.120747673625916e-07, + "loss": 0.6176275253295899, + "step": 5200 + }, + { + "epoch": 0.8349389852280026, + "eval_loss": 0.5767529010772705, + "eval_runtime": 115.3879, + "eval_samples_per_second": 17.532, + "eval_steps_per_second": 4.385, + "eval_token_acc": 0.8026373967407536, + "step": 5200 + }, + { + "epoch": 0.8365446371226718, + "grad_norm": 1.6616720449719025, + "learning_rate": 6.986190387658909e-07, + "loss": 0.6671013832092285, + "step": 5210 + }, + { + "epoch": 0.838150289017341, + "grad_norm": 1.7360372162228797, + "learning_rate": 6.852821042586183e-07, + "loss": 0.6652707099914551, + "step": 5220 + }, + { + "epoch": 0.8397559409120102, + "grad_norm": 1.6228551388753156, + "learning_rate": 6.720643321757348e-07, + "loss": 0.6581364631652832, + "step": 5230 + }, + { + "epoch": 0.8413615928066795, + "grad_norm": 1.722082150924103, + "learning_rate": 6.589660875612147e-07, + "loss": 0.6475844383239746, + "step": 5240 + }, + { + "epoch": 0.8429672447013488, + "grad_norm": 2.116679273977666, + "learning_rate": 6.459877321579628e-07, + "loss": 0.6615125656127929, + "step": 5250 + }, + { + "epoch": 0.844572896596018, + "grad_norm": 1.8762326334249044, + "learning_rate": 6.33129624397823e-07, + "loss": 0.6376981258392334, + "step": 5260 + }, + { + "epoch": 0.8461785484906872, + "grad_norm": 1.5775860849763117, + "learning_rate": 6.203921193916812e-07, + "loss": 0.5770237922668457, + "step": 5270 + }, + { + "epoch": 0.8477842003853564, + "grad_norm": 1.9029054905764342, + "learning_rate": 6.077755689196574e-07, + "loss": 0.6483820915222168, + "step": 5280 + }, + { + "epoch": 0.8493898522800257, + "grad_norm": 1.8325196688318919, + "learning_rate": 5.952803214213887e-07, + "loss": 0.6220078468322754, + "step": 5290 + }, + { + "epoch": 0.8509955041746949, + "grad_norm": 1.6696224296793218, + "learning_rate": 5.829067219864099e-07, + "loss": 0.666869831085205, + "step": 5300 + }, + { + "epoch": 0.8526011560693642, + "grad_norm": 1.5010642399247371, + "learning_rate": 5.706551123446175e-07, + "loss": 0.5817923545837402, + "step": 5310 + }, + { + "epoch": 0.8542068079640334, + "grad_norm": 1.6418838210429256, + "learning_rate": 5.585258308568381e-07, + "loss": 0.6304399490356445, + "step": 5320 + }, + { + "epoch": 0.8558124598587026, + "grad_norm": 1.9627540497013132, + "learning_rate": 5.465192125054769e-07, + "loss": 0.6063569068908692, + "step": 5330 + }, + { + "epoch": 0.8574181117533719, + "grad_norm": 1.8273506016319814, + "learning_rate": 5.346355888852767e-07, + "loss": 0.610871696472168, + "step": 5340 + }, + { + "epoch": 0.8590237636480411, + "grad_norm": 1.7712709881033286, + "learning_rate": 5.22875288194144e-07, + "loss": 0.5931038856506348, + "step": 5350 + }, + { + "epoch": 0.8606294155427103, + "grad_norm": 1.690149934214249, + "learning_rate": 5.112386352241017e-07, + "loss": 0.6216620922088623, + "step": 5360 + }, + { + "epoch": 0.8622350674373795, + "grad_norm": 1.57401450836848, + "learning_rate": 4.997259513523079e-07, + "loss": 0.6697893619537354, + "step": 5370 + }, + { + "epoch": 0.8638407193320488, + "grad_norm": 1.766758207988638, + "learning_rate": 4.883375545321845e-07, + "loss": 0.6087721347808838, + "step": 5380 + }, + { + "epoch": 0.8654463712267181, + "grad_norm": 1.4690060560601916, + "learning_rate": 4.770737592846375e-07, + "loss": 0.6009274482727051, + "step": 5390 + }, + { + "epoch": 0.8670520231213873, + "grad_norm": 1.6074705175725752, + "learning_rate": 4.6593487668936565e-07, + "loss": 0.6610477924346924, + "step": 5400 + }, + { + "epoch": 0.8670520231213873, + "eval_loss": 0.5729334950447083, + "eval_runtime": 116.3697, + "eval_samples_per_second": 17.384, + "eval_steps_per_second": 4.348, + "eval_token_acc": 0.8028926771237075, + "step": 5400 + }, + { + "epoch": 0.8686576750160565, + "grad_norm": 1.57876027358775, + "learning_rate": 4.5492121437627433e-07, + "loss": 0.6015999794006348, + "step": 5410 + }, + { + "epoch": 0.8702633269107257, + "grad_norm": 1.7832820730897672, + "learning_rate": 4.440330765169765e-07, + "loss": 0.5984902381896973, + "step": 5420 + }, + { + "epoch": 0.871868978805395, + "grad_norm": 1.6137384209920154, + "learning_rate": 4.3327076381639357e-07, + "loss": 0.6636902809143066, + "step": 5430 + }, + { + "epoch": 0.8734746307000643, + "grad_norm": 1.8766192417776106, + "learning_rate": 4.226345735044485e-07, + "loss": 0.6453081130981445, + "step": 5440 + }, + { + "epoch": 0.8750802825947335, + "grad_norm": 1.7263370713948474, + "learning_rate": 4.121247993278621e-07, + "loss": 0.6501432418823242, + "step": 5450 + }, + { + "epoch": 0.8766859344894027, + "grad_norm": 1.6370094742186325, + "learning_rate": 4.0174173154203356e-07, + "loss": 0.6349458694458008, + "step": 5460 + }, + { + "epoch": 0.8782915863840719, + "grad_norm": 1.6338222328943681, + "learning_rate": 3.9148565690302896e-07, + "loss": 0.6203731536865235, + "step": 5470 + }, + { + "epoch": 0.8798972382787412, + "grad_norm": 1.5790217288544453, + "learning_rate": 3.813568586596611e-07, + "loss": 0.5949804306030273, + "step": 5480 + }, + { + "epoch": 0.8815028901734104, + "grad_norm": 1.39406428632037, + "learning_rate": 3.7135561654566497e-07, + "loss": 0.6562387466430664, + "step": 5490 + }, + { + "epoch": 0.8831085420680796, + "grad_norm": 1.6638397207650397, + "learning_rate": 3.6148220677197364e-07, + "loss": 0.6203203201293945, + "step": 5500 + }, + { + "epoch": 0.8847141939627489, + "grad_norm": 1.4019127036161603, + "learning_rate": 3.5173690201909084e-07, + "loss": 0.6110305786132812, + "step": 5510 + }, + { + "epoch": 0.8863198458574181, + "grad_norm": 1.7918708664402199, + "learning_rate": 3.4211997142955756e-07, + "loss": 0.6375399589538574, + "step": 5520 + }, + { + "epoch": 0.8879254977520874, + "grad_norm": 1.9616932846212223, + "learning_rate": 3.326316806005209e-07, + "loss": 0.6158774375915528, + "step": 5530 + }, + { + "epoch": 0.8895311496467566, + "grad_norm": 1.8162630013198795, + "learning_rate": 3.2327229157639915e-07, + "loss": 0.6023660659790039, + "step": 5540 + }, + { + "epoch": 0.8911368015414258, + "grad_norm": 1.5475924483567713, + "learning_rate": 3.1404206284164295e-07, + "loss": 0.6220304489135742, + "step": 5550 + }, + { + "epoch": 0.892742453436095, + "grad_norm": 1.780906089201377, + "learning_rate": 3.0494124931359834e-07, + "loss": 0.605389404296875, + "step": 5560 + }, + { + "epoch": 0.8943481053307643, + "grad_norm": 1.6714265896053035, + "learning_rate": 2.959701023354644e-07, + "loss": 0.6368635177612305, + "step": 5570 + }, + { + "epoch": 0.8959537572254336, + "grad_norm": 1.5909364323881428, + "learning_rate": 2.871288696693564e-07, + "loss": 0.6405464172363281, + "step": 5580 + }, + { + "epoch": 0.8975594091201028, + "grad_norm": 1.8531889188206607, + "learning_rate": 2.7841779548945626e-07, + "loss": 0.5804450511932373, + "step": 5590 + }, + { + "epoch": 0.899165061014772, + "grad_norm": 1.5833984517267605, + "learning_rate": 2.698371203752753e-07, + "loss": 0.6217172145843506, + "step": 5600 + }, + { + "epoch": 0.899165061014772, + "eval_loss": 0.5752645134925842, + "eval_runtime": 115.6265, + "eval_samples_per_second": 17.496, + "eval_steps_per_second": 4.376, + "eval_token_acc": 0.803150460255514, + "step": 5600 + }, + { + "epoch": 0.9007707129094412, + "grad_norm": 1.6237404405424622, + "learning_rate": 2.613870813050051e-07, + "loss": 0.5858840465545654, + "step": 5610 + }, + { + "epoch": 0.9023763648041104, + "grad_norm": 1.8525333922294798, + "learning_rate": 2.53067911648977e-07, + "loss": 0.6315989971160889, + "step": 5620 + }, + { + "epoch": 0.9039820166987798, + "grad_norm": 1.768138904733902, + "learning_rate": 2.4487984116321474e-07, + "loss": 0.6300556659698486, + "step": 5630 + }, + { + "epoch": 0.905587668593449, + "grad_norm": 1.7161529988705222, + "learning_rate": 2.368230959830875e-07, + "loss": 0.6011830329895019, + "step": 5640 + }, + { + "epoch": 0.9071933204881182, + "grad_norm": 1.600603600858998, + "learning_rate": 2.2889789861706868e-07, + "loss": 0.6122558116912842, + "step": 5650 + }, + { + "epoch": 0.9087989723827874, + "grad_norm": 1.928618788414808, + "learning_rate": 2.211044679405877e-07, + "loss": 0.5967905044555664, + "step": 5660 + }, + { + "epoch": 0.9104046242774566, + "grad_norm": 1.5630153842153907, + "learning_rate": 2.1344301918998555e-07, + "loss": 0.6117796421051025, + "step": 5670 + }, + { + "epoch": 0.9120102761721259, + "grad_norm": 1.6679276269534378, + "learning_rate": 2.059137639565717e-07, + "loss": 0.6195221900939941, + "step": 5680 + }, + { + "epoch": 0.9136159280667951, + "grad_norm": 1.638132151814036, + "learning_rate": 1.9851691018077824e-07, + "loss": 0.6240126609802246, + "step": 5690 + }, + { + "epoch": 0.9152215799614644, + "grad_norm": 1.6864448316312712, + "learning_rate": 1.9125266214642e-07, + "loss": 0.6728627681732178, + "step": 5700 + }, + { + "epoch": 0.9168272318561336, + "grad_norm": 1.8998671947688817, + "learning_rate": 1.8412122047505032e-07, + "loss": 0.6206884384155273, + "step": 5710 + }, + { + "epoch": 0.9184328837508028, + "grad_norm": 1.5534987287794495, + "learning_rate": 1.7712278212042134e-07, + "loss": 0.6320960998535157, + "step": 5720 + }, + { + "epoch": 0.9200385356454721, + "grad_norm": 1.681299514817593, + "learning_rate": 1.7025754036304466e-07, + "loss": 0.6238042831420898, + "step": 5730 + }, + { + "epoch": 0.9216441875401413, + "grad_norm": 1.6265592202582009, + "learning_rate": 1.6352568480485277e-07, + "loss": 0.6073272228240967, + "step": 5740 + }, + { + "epoch": 0.9232498394348105, + "grad_norm": 1.8025202193600034, + "learning_rate": 1.5692740136396324e-07, + "loss": 0.5922677993774415, + "step": 5750 + }, + { + "epoch": 0.9248554913294798, + "grad_norm": 1.791643260329241, + "learning_rate": 1.5046287226954394e-07, + "loss": 0.6612445831298828, + "step": 5760 + }, + { + "epoch": 0.926461143224149, + "grad_norm": 1.6626324219461264, + "learning_rate": 1.4413227605677983e-07, + "loss": 0.6229853630065918, + "step": 5770 + }, + { + "epoch": 0.9280667951188183, + "grad_norm": 1.679861943616916, + "learning_rate": 1.379357875619436e-07, + "loss": 0.6264891147613525, + "step": 5780 + }, + { + "epoch": 0.9296724470134875, + "grad_norm": 1.721027746101911, + "learning_rate": 1.3187357791756504e-07, + "loss": 0.6237311363220215, + "step": 5790 + }, + { + "epoch": 0.9312780989081567, + "grad_norm": 1.6107596914991866, + "learning_rate": 1.2594581454770772e-07, + "loss": 0.5913224697113038, + "step": 5800 + }, + { + "epoch": 0.9312780989081567, + "eval_loss": 0.5708259344100952, + "eval_runtime": 117.1262, + "eval_samples_per_second": 17.272, + "eval_steps_per_second": 4.32, + "eval_token_acc": 0.8033098019324558, + "step": 5800 + }, + { + "epoch": 0.9328837508028259, + "grad_norm": 1.7093807551813902, + "learning_rate": 1.2015266116334135e-07, + "loss": 0.6007876396179199, + "step": 5810 + }, + { + "epoch": 0.9344894026974951, + "grad_norm": 1.62030645952606, + "learning_rate": 1.1449427775782396e-07, + "loss": 0.5883060932159424, + "step": 5820 + }, + { + "epoch": 0.9360950545921645, + "grad_norm": 2.4493142088797346, + "learning_rate": 1.0897082060247976e-07, + "loss": 0.6372334003448487, + "step": 5830 + }, + { + "epoch": 0.9377007064868337, + "grad_norm": 1.6352836346158377, + "learning_rate": 1.0358244224228764e-07, + "loss": 0.6133495330810547, + "step": 5840 + }, + { + "epoch": 0.9393063583815029, + "grad_norm": 1.5838720024708473, + "learning_rate": 9.832929149166503e-08, + "loss": 0.5925948143005371, + "step": 5850 + }, + { + "epoch": 0.9409120102761721, + "grad_norm": 1.6297564720837419, + "learning_rate": 9.32115134303574e-08, + "loss": 0.6138250350952148, + "step": 5860 + }, + { + "epoch": 0.9425176621708413, + "grad_norm": 1.6701533347653996, + "learning_rate": 8.822924939943523e-08, + "loss": 0.6651721000671387, + "step": 5870 + }, + { + "epoch": 0.9441233140655106, + "grad_norm": 1.5684211124230398, + "learning_rate": 8.338263699738668e-08, + "loss": 0.5427798271179199, + "step": 5880 + }, + { + "epoch": 0.9457289659601799, + "grad_norm": 1.9455396681578898, + "learning_rate": 7.867181007631897e-08, + "loss": 0.6322480201721191, + "step": 5890 + }, + { + "epoch": 0.9473346178548491, + "grad_norm": 1.7580700885542513, + "learning_rate": 7.409689873826232e-08, + "loss": 0.6054346561431885, + "step": 5900 + }, + { + "epoch": 0.9489402697495183, + "grad_norm": 1.9972458734560363, + "learning_rate": 6.965802933157573e-08, + "loss": 0.639186429977417, + "step": 5910 + }, + { + "epoch": 0.9505459216441875, + "grad_norm": 1.7180914027664866, + "learning_rate": 6.535532444745862e-08, + "loss": 0.6357224464416504, + "step": 5920 + }, + { + "epoch": 0.9521515735388568, + "grad_norm": 1.9042172337567278, + "learning_rate": 6.118890291656355e-08, + "loss": 0.6573823928833008, + "step": 5930 + }, + { + "epoch": 0.953757225433526, + "grad_norm": 1.7870529920045328, + "learning_rate": 5.7158879805716e-08, + "loss": 0.573056697845459, + "step": 5940 + }, + { + "epoch": 0.9553628773281952, + "grad_norm": 1.840970426056217, + "learning_rate": 5.32653664147359e-08, + "loss": 0.5925376892089844, + "step": 5950 + }, + { + "epoch": 0.9569685292228645, + "grad_norm": 1.6478271540265557, + "learning_rate": 4.950847027336336e-08, + "loss": 0.6077459335327149, + "step": 5960 + }, + { + "epoch": 0.9585741811175337, + "grad_norm": 1.6679108958537707, + "learning_rate": 4.588829513828996e-08, + "loss": 0.6572588443756103, + "step": 5970 + }, + { + "epoch": 0.960179833012203, + "grad_norm": 1.5893205392993202, + "learning_rate": 4.2404940990292135e-08, + "loss": 0.5608067512512207, + "step": 5980 + }, + { + "epoch": 0.9617854849068722, + "grad_norm": 1.725453941125604, + "learning_rate": 3.90585040314706e-08, + "loss": 0.5632122039794922, + "step": 5990 + }, + { + "epoch": 0.9633911368015414, + "grad_norm": 1.7376641214713817, + "learning_rate": 3.584907668259308e-08, + "loss": 0.6706958770751953, + "step": 6000 + }, + { + "epoch": 0.9633911368015414, + "eval_loss": 0.5741922855377197, + "eval_runtime": 116.3672, + "eval_samples_per_second": 17.385, + "eval_steps_per_second": 4.348, + "eval_token_acc": 0.8032964539385758, + "step": 6000 + }, + { + "epoch": 0.9649967886962106, + "grad_norm": 1.5435197879461982, + "learning_rate": 3.2776747580541835e-08, + "loss": 0.6008576393127442, + "step": 6010 + }, + { + "epoch": 0.9666024405908799, + "grad_norm": 1.6555958936202548, + "learning_rate": 2.984160157586735e-08, + "loss": 0.6182814598083496, + "step": 6020 + }, + { + "epoch": 0.9682080924855492, + "grad_norm": 1.5683265883313169, + "learning_rate": 2.7043719730441288e-08, + "loss": 0.6090614318847656, + "step": 6030 + }, + { + "epoch": 0.9698137443802184, + "grad_norm": 1.6330006682221332, + "learning_rate": 2.4383179315222717e-08, + "loss": 0.5801780223846436, + "step": 6040 + }, + { + "epoch": 0.9714193962748876, + "grad_norm": 1.9282013764770385, + "learning_rate": 2.186005380811873e-08, + "loss": 0.6230826377868652, + "step": 6050 + }, + { + "epoch": 0.9730250481695568, + "grad_norm": 1.612559249065291, + "learning_rate": 1.9474412891959395e-08, + "loss": 0.6020002365112305, + "step": 6060 + }, + { + "epoch": 0.974630700064226, + "grad_norm": 1.6913118413453614, + "learning_rate": 1.7226322452572055e-08, + "loss": 0.6456441402435302, + "step": 6070 + }, + { + "epoch": 0.9762363519588954, + "grad_norm": 1.5870625483228484, + "learning_rate": 1.5115844576960027e-08, + "loss": 0.6072479248046875, + "step": 6080 + }, + { + "epoch": 0.9778420038535646, + "grad_norm": 1.4885864603630632, + "learning_rate": 1.3143037551590076e-08, + "loss": 0.6364302635192871, + "step": 6090 + }, + { + "epoch": 0.9794476557482338, + "grad_norm": 1.7423098004833633, + "learning_rate": 1.1307955860781483e-08, + "loss": 0.630467700958252, + "step": 6100 + }, + { + "epoch": 0.981053307642903, + "grad_norm": 1.7819265377180369, + "learning_rate": 9.610650185202242e-09, + "loss": 0.6165467262268066, + "step": 6110 + }, + { + "epoch": 0.9826589595375722, + "grad_norm": 1.493769172139119, + "learning_rate": 8.051167400467408e-09, + "loss": 0.6086311340332031, + "step": 6120 + }, + { + "epoch": 0.9842646114322415, + "grad_norm": 1.4828619626593085, + "learning_rate": 6.629550575847355e-09, + "loss": 0.6316121101379395, + "step": 6130 + }, + { + "epoch": 0.9858702633269107, + "grad_norm": 1.6623261401956104, + "learning_rate": 5.3458389730765006e-09, + "loss": 0.6523394107818603, + "step": 6140 + }, + { + "epoch": 0.98747591522158, + "grad_norm": 1.7908448179525431, + "learning_rate": 4.200068045269179e-09, + "loss": 0.6246747016906739, + "step": 6150 + }, + { + "epoch": 0.9890815671162492, + "grad_norm": 1.7502121618975803, + "learning_rate": 3.1922694359404207e-09, + "loss": 0.6109015464782714, + "step": 6160 + }, + { + "epoch": 0.9906872190109184, + "grad_norm": 1.9849207865849259, + "learning_rate": 2.3224709781322075e-09, + "loss": 0.5936941623687744, + "step": 6170 + }, + { + "epoch": 0.9922928709055877, + "grad_norm": 1.5565505599418075, + "learning_rate": 1.5906966936457545e-09, + "loss": 0.6239344120025635, + "step": 6180 + }, + { + "epoch": 0.9938985228002569, + "grad_norm": 1.8598198518642268, + "learning_rate": 9.969667923764859e-10, + "loss": 0.6211441993713379, + "step": 6190 + }, + { + "epoch": 0.9955041746949261, + "grad_norm": 1.924048487204165, + "learning_rate": 5.412976717561469e-10, + "loss": 0.5937876224517822, + "step": 6200 + }, + { + "epoch": 0.9955041746949261, + "eval_loss": 0.5703438520431519, + "eval_runtime": 116.1219, + "eval_samples_per_second": 17.421, + "eval_steps_per_second": 4.357, + "eval_token_acc": 0.8032922826904884, + "step": 6200 + }, + { + "epoch": 0.9971098265895953, + "grad_norm": 1.6927629675679943, + "learning_rate": 2.2370191630149974e-10, + "loss": 0.5909645557403564, + "step": 6210 + }, + { + "epoch": 0.9987154784842646, + "grad_norm": 1.6900193485291426, + "learning_rate": 4.418829726460239e-11, + "loss": 0.6455328941345215, + "step": 6220 + }, + { + "epoch": 1.0, + "eval_loss": 0.5718016028404236, + "eval_runtime": 116.7597, + "eval_samples_per_second": 17.326, + "eval_steps_per_second": 4.334, + "eval_token_acc": 0.8033565199110356, + "step": 6228 + } + ], + "logging_steps": 10, + "max_steps": 6228, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 264777532440576.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..7df76e1 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2bc3b695ab287e3d2676c86ee34633cb4218eec1a80343f59f0a3eb2c967045 +size 9425 diff --git a/vocab.json b/vocab.json new file mode 100644 index 0000000..6c49fc6 --- /dev/null +++ b/vocab.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910 +size 2776833