commit a1080453d51e4e0072db6c7169146e81ae24d20f Author: ModelHub XC Date: Sun Jul 26 13:32:07 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: stratosphere/qwen2.5-1.5b-slips-immune-unified Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..9cd0fca --- /dev/null +++ b/README.md @@ -0,0 +1,282 @@ +--- +language: +- en +license: apache-2.0 +tags: +- text-generation +- transformers +- safetensors +- network-security +- ids +- slips +- summarization +- cause-analysis +- risk-assessment +- cybersecurity +- lora +- sft +- trl +- unsloth +- qwen2 +base_model: unsloth/Qwen2.5-1.5B-Instruct +datasets: +- stratosphere/immune-unified-sft-dataset +pipeline_tag: text-generation +--- + +# Qwen2.5-1.5B — Slips IDS Unified Security Analyst (v2) + +## Model Description + +A fine-tuned version of [Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct) specialized for **three complementary security analysis tasks** on network incidents from [Slips IDS](https://github.com/stratosphereips/StratosphereLinuxIPS) — all in a single adapter: + +1. **Summarization** — translating technical Slips DAG alert logs into clear, human-readable incident summaries with per-event severity labels (CRITICAL / HIGH / MEDIUM / LOW / INFO) +2. **Cause Analysis** — identifying the likely cause (malicious activity, misconfiguration, or legitimate behavior) with structured reasoning +3. **Risk Assessment** — producing calibrated risk level, business impact, likelihood of malicious activity, and investigation priority + +Slips is a network intrusion detection system that generates DAG-structured alert logs — chains of related security events per source IP per time window. This unified model handles the full analyst pipeline in one inference call or as separate targeted queries. + +This model merges the capabilities of [stratosphere/qwen2.5-1.5b-slips-immune-summarization](https://huggingface.co/stratosphere/qwen2.5-1.5b-slips-immune-summarization) and [stratosphere/qwen2.5-1.5b-slips-immune-risk](https://huggingface.co/stratosphere/qwen2.5-1.5b-slips-immune-risk) into a single fine-tuned adapter trained jointly on all three tasks. + +--- + +## Quick Start + +### Ollama (Recommended) +```bash +ollama run harpomaxx/qwen2.5-1.5b-slips-immune-unified-v2 +``` + +### Python (Transformers) +```python +from transformers import AutoTokenizer, AutoModelForCausalLM +import torch + +model_id = "harpomaxx/qwen2.5-1.5b-slips-immune-unified-v2" +tokenizer = AutoTokenizer.from_pretrained(model_id) +model = AutoModelForCausalLM.from_pretrained( + model_id, torch_dtype=torch.bfloat16, device_map="auto" +) + +# --- Task 1: Summarization --- +summary_prompt = """You are a security analyst. Your task is to translate technical security events into clear, concise, human-readable summaries and assess their severity. + +INCIDENT METADATA: +- Incident ID: {incident_id} +- Source IP: {source_ip} +- Timewindow: {timewindow} +- Accumulated Threat Level: {threat_level} +- Time Range: {start} to {end} +- Total Events: {count} + +RAW EVENTS: +{dag_analysis} + +YOUR TASK: +1. Transform technical event descriptions into clear, readable summaries +2. Group identical or similar events +3. Assess severity (CRITICAL/HIGH/MEDIUM/LOW/INFO) +4. Calculate overall severity breakdown + +OUTPUT FORMAT: +============================================================ +Incident: +Source IP: | Timewindow: +Timeline: to +Threat Level: | Events: + +• HH:MM-HH:MM - [Your clear grouped summary] [SEVERITY] +• HH:MM - [Your clear summary] [SEVERITY] + +Total Evidence: events +Severity breakdown: [e.g., "High: 5, Medium: 3, Info: 2"]""" + +# --- Task 2: Cause Analysis --- +cause_prompt = """You are a cybersecurity analyst. Analyze the following network security incident and provide a structured analysis of possible causes. + +INCIDENT METADATA: +- Incident ID: {incident_id} +- Source IP: {source_ip} +- Accumulated Threat Level: {threat_level} + +SECURITY EVIDENCE: +{dag_analysis} + +Output Requirements: +- Respond with ONLY the analysis content + +**Possible Causes:** + +**1. Malicious Activity:** +• [Specific attack technique] + +**2. Legitimate Activity:** +• [Benign operational cause] + +**3. Misconfigurations:** +• [Technical misconfigurations] + +**Conclusion:** [Assessment of most likely cause category]""" + +# --- Task 3: Risk Assessment --- +risk_prompt = """You are a cybersecurity analyst. Analyze the following network security incident and provide a structured risk assessment. + +INCIDENT METADATA: +- Incident ID: {incident_id} +- Source IP: {source_ip} +- Accumulated Threat Level: {threat_level} + +SECURITY EVIDENCE: +{dag_analysis} + +**Risk Level:** [Critical/High/Medium/Low] + +**Justification:** [Technical justification] + +**Business Impact:** [Single clear sentence describing business effect] + +**Likelihood of Malicious Activity:** [High/Medium/Low] - [Brief rationale] + +**Investigation Priority:** [Immediate/High/Medium/Low] - [Brief justification]""" + +def run_task(prompt): + messages = [{"role": "user", "content": prompt}] + input_ids = tokenizer.apply_chat_template( + messages, return_tensors="pt", add_generation_prompt=True + ).to(model.device) + output = model.generate(input_ids, max_new_tokens=512, do_sample=False) + return tokenizer.decode(output[0][input_ids.shape[1]:], skip_special_tokens=True) +``` + +--- + +## Training Details + +### Dataset + +- **Source**: 750 incidents from real Slips IDS network captures (675 train / 75 eval incidents) +- **Tasks**: Three tasks per incident — summarization (S), cause analysis (A), risk assessment (B) — interleaved +- **Responses**: 4 model responses per incident per task (GPT-4o, GPT-4o-mini, Qwen2.5 3B, Qwen2.5 1.5B) +- **Selection**: Best-of-N — highest-scoring response selected via LLM-as-judge +- **Filtering**: Responses with judge score < 4 discarded +- **Split**: 2195 train / 225 eval records (augmented with 85 risk-only extra samples, seed=42) +- **Dataset**: [stratosphere/immune-unified-sft-dataset](https://huggingface.co/datasets/stratosphere/immune-unified-sft-dataset) + +### Training Procedure + +| Parameter | Value | +|-----------|-------| +| Base Model | `unsloth/Qwen2.5-1.5B-Instruct` | +| Training Method | SFT (Supervised Fine-Tuning) | +| Framework | Unsloth + TRL SFTTrainer | +| LoRA Rank (r) | 128 | +| LoRA Alpha | 128 | +| LoRA Dropout | 0.0 | +| RSLoRA | Enabled (required at r=64) | +| LoRA Targets | q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj | +| Sequence Length | 4096 | +| Batch Size | 1 (effective: 16 via gradient accumulation) | +| Learning Rate | 2e-5 | +| LR Scheduler | Cosine | +| Warmup Steps | 30 | +| Weight Decay | 0.01 | +| Epochs | 2 | +| Optimizer | adamw_8bit | +| Precision | BF16 | +| Quantization | 4bit (QLoRA) | +| Hardware | A100 80GB MiG 20GB slice | + +### Training Results + +| Step | Epoch | Eval Loss | +|------|-------|-----------| +| 50 | 0.57 | 0.8047 | +| 100 | 1.12 | 0.7594 | +| 150 | 1.69 | 0.7426 | +| 200 | 2.25 | 0.7327 | +| **250** | **2.82** | **0.7293** ← best | + +Eval loss decreased monotonically across all checkpoints with no sign of overfitting. + +--- + +## Evaluation Results + +### Summarization Task + +Evaluated on 47 held-out Slips IDS incidents using `gpt-oss-120b` as an independent LLM-as-judge. + +| Rank | Model | Avg Score | Win Rate | +|------|-------|-----------|----------| +| 1 | GPT-4o-mini | 6.89/10 | 42.6% | +| 2 | GPT-4o | 5.87/10 | 29.8% | +| **3** | **Qwen2.5-1.5B (finetuned)** | **4.70/10** | **19.1%** | +| 4 | Qwen2.5 3B (baseline) | 4.57/10 | 8.5% | +| 5 | Qwen2.5 1B (baseline) | 3.36/10 | 0.0% | + +The finetuned 1.5B model beats both untuned baselines and achieves a 19.1% win rate — higher than the 3B baseline. + +### Cause Analysis & Risk Assessment Tasks + +Evaluated on 67 held-out Slips IDS incidents. + +| Rank | Model | Avg Cause Score | Avg Risk Score | Win Rate | +|------|-------|-----------------|----------------|----------| +| 1 | GPT-4o | 15.33 | 11.99 | 40.3% | +| **2** | **Qwen2.5-1.5B (finetuned)** | **15.58** | **10.27** | **37.3%** | +| 3 | GPT-4o-mini | 15.31 | 11.63 | 19.4% | +| 4 | Qwen2.5 1.5B (baseline) | 9.15 | 8.79 | 3.0% | +| 5 | Qwen2.5 3B (baseline) | 7.40 | 9.61 | 0.0% | + +**Key Finding**: The finetuned model is nearly tied with GPT-4o overall and **beats GPT-4o on cause analysis** (15.58 vs 15.33), at a fraction of the inference cost. + +--- + +## Known Limitations + +- **Context window**: Performance degrades on incidents with ≥500 events where DAG token counts exceed the 4096-token limit. Complex incidents are truncated. +- **Risk calibration**: The model is stronger at identifying causes than calibrating risk levels (cause score 15.58 vs risk score 10.27). +- **Normal traffic**: Summarization accuracy on normal (benign) traffic is lower than on incident traffic. +- **Domain**: Trained exclusively on Slips IDS logs — not suitable for other IDS formats or general security tasks. + +--- + +## Intended Use + +- Automated triage of Slips IDS alerts for security analysts +- Full pipeline: summarize → analyze cause → assess risk, in a single model +- First-pass analysis of network incident logs as input to downstream reporting or ticketing workflows +- Edge/on-premises deployment (RPi5, low-resource servers) via GGUF quantization + +## Out-of-Scope Use + +- General-purpose chat or instruction following +- Security domains outside Slips IDS / network intrusion detection +- Non-English inputs + +--- + +## Model Details + +- **Model Size**: 1.5B parameters +- **Tensor Type**: BF16 +- **License**: Apache-2.0 + +--- + +## Citation + +```bibtex +@misc{qwen2.5-1.5b-slips-unified, + title = {Qwen2.5-1.5B fine-tuned for unified Slips IDS security analysis}, + author = {Stratosphere Laboratory, CTU Prague}, + year = {2026}, + howpublished = {\url{https://huggingface.co/harpomaxx/qwen2.5-1.5b-slips-immune-unified-v2}} +} +``` + +--- + +## Acknowledgments + +Supported by the **NLnet Foundation** as part of the IMMUNE project, promoting open internet standards and open source software. diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..642e597 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,53 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..4dcbf72 --- /dev/null +++ b/config.json @@ -0,0 +1,62 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "torch_dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151665, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "unsloth_fixed": true, + "unsloth_version": "2026.6.1", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..5f32a68 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,14 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "max_length": 32768, + "pad_token_id": 151665, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.5.0" +} diff --git a/lora_adapter/README.md b/lora_adapter/README.md new file mode 100644 index 0000000..05d54c2 --- /dev/null +++ b/lora_adapter/README.md @@ -0,0 +1,63 @@ +--- +base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +library_name: peft +model_name: qwen_unified_finetuned_v2 +tags: +- base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +- lora +- sft +- transformers +- trl +- unsloth +licence: license +pipeline_tag: text-generation +--- + +# Model Card for qwen_unified_finetuned_v2 + +This model is a fine-tuned version of [unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit](https://huggingface.co/unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + +This model was trained with SFT. + +### Framework versions + +- PEFT 0.19.1 +- TRL: 0.24.0 +- Transformers: 5.5.0 +- Pytorch: 2.10.0 +- Datasets: 4.3.0 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@misc{vonwerra2022trl, + title = {{TRL: Transformer Reinforcement Learning}}, + author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec}, + year = 2020, + journal = {GitHub repository}, + publisher = {GitHub}, + howpublished = {\url{https://github.com/huggingface/trl}} +} +``` \ No newline at end of file diff --git a/lora_adapter/adapter_config.json b/lora_adapter/adapter_config.json new file mode 100644 index 0000000..cdc21db --- /dev/null +++ b/lora_adapter/adapter_config.json @@ -0,0 +1,52 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": { + "base_model_class": "Qwen2ForCausalLM", + "parent_library": "transformers.models.qwen2.modeling_qwen2", + "unsloth_fixed": true + }, + "base_model_name_or_path": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "k_proj", + "q_proj", + "gate_proj", + "v_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": true +} \ No newline at end of file diff --git a/lora_adapter/adapter_model.safetensors b/lora_adapter/adapter_model.safetensors new file mode 100644 index 0000000..af7e749 --- /dev/null +++ b/lora_adapter/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46babf050836e50ce540a66fa0d2fd0b2c7914b8b902b8e46ccd48685d72c6a0 +size 590925768 diff --git a/lora_adapter/chat_template.jinja b/lora_adapter/chat_template.jinja new file mode 100644 index 0000000..642e597 --- /dev/null +++ b/lora_adapter/chat_template.jinja @@ -0,0 +1,53 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/lora_adapter/checkpoint-150/README.md b/lora_adapter/checkpoint-150/README.md new file mode 100644 index 0000000..a4c765d --- /dev/null +++ b/lora_adapter/checkpoint-150/README.md @@ -0,0 +1,210 @@ +--- +base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +- lora +- sft +- transformers +- trl +- unsloth +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/lora_adapter/checkpoint-150/adapter_config.json b/lora_adapter/checkpoint-150/adapter_config.json new file mode 100644 index 0000000..cdc21db --- /dev/null +++ b/lora_adapter/checkpoint-150/adapter_config.json @@ -0,0 +1,52 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": { + "base_model_class": "Qwen2ForCausalLM", + "parent_library": "transformers.models.qwen2.modeling_qwen2", + "unsloth_fixed": true + }, + "base_model_name_or_path": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "k_proj", + "q_proj", + "gate_proj", + "v_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": true +} \ No newline at end of file diff --git a/lora_adapter/checkpoint-150/adapter_model.safetensors b/lora_adapter/checkpoint-150/adapter_model.safetensors new file mode 100644 index 0000000..b801b22 --- /dev/null +++ b/lora_adapter/checkpoint-150/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f56254b3b5e5fc5b6b2a5adc82825a169d3e0f8b933d22c18868ee361d9bf1b6 +size 590925768 diff --git a/lora_adapter/checkpoint-150/chat_template.jinja b/lora_adapter/checkpoint-150/chat_template.jinja new file mode 100644 index 0000000..642e597 --- /dev/null +++ b/lora_adapter/checkpoint-150/chat_template.jinja @@ -0,0 +1,53 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/lora_adapter/checkpoint-150/optimizer.pt b/lora_adapter/checkpoint-150/optimizer.pt new file mode 100644 index 0000000..fd6560b --- /dev/null +++ b/lora_adapter/checkpoint-150/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fddcde1532688ccd38684526a3bc395b54fd4f6ffa3750f55b463274080fd04 +size 300517573 diff --git a/lora_adapter/checkpoint-150/rng_state.pth b/lora_adapter/checkpoint-150/rng_state.pth new file mode 100644 index 0000000..e92cf8f --- /dev/null +++ b/lora_adapter/checkpoint-150/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:718a0f3db00824213036a2c0441849791319b7d9cf189065873bb26a7020738e +size 14645 diff --git a/lora_adapter/checkpoint-150/scheduler.pt b/lora_adapter/checkpoint-150/scheduler.pt new file mode 100644 index 0000000..757ca4b --- /dev/null +++ b/lora_adapter/checkpoint-150/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00556b2b78828c5f99c9ad58ab04d133f48e13e9f80bd5511420df5dfd85d00f +size 1465 diff --git a/lora_adapter/checkpoint-150/tokenizer.json b/lora_adapter/checkpoint-150/tokenizer.json new file mode 100644 index 0000000..5340d81 --- /dev/null +++ b/lora_adapter/checkpoint-150/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd5948af71b4f56cf697f7580814c7ce8b80595ef985544efcacf716126a2e31 +size 11422356 diff --git a/lora_adapter/checkpoint-150/tokenizer_config.json b/lora_adapter/checkpoint-150/tokenizer_config.json new file mode 100644 index 0000000..d3dac53 --- /dev/null +++ b/lora_adapter/checkpoint-150/tokenizer_config.json @@ -0,0 +1,201 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 32768, + "pad_token": "<|PAD_TOKEN|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151657": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151658": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151665": { + "content": "<|PAD_TOKEN|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + } +} diff --git a/lora_adapter/checkpoint-150/trainer_state.json b/lora_adapter/checkpoint-150/trainer_state.json new file mode 100644 index 0000000..8572bc7 --- /dev/null +++ b/lora_adapter/checkpoint-150/trainer_state.json @@ -0,0 +1,1108 @@ +{ + "best_global_step": 150, + "best_metric": 0.6550542712211609, + "best_model_checkpoint": "./qwen_unified_finetuned_v2/checkpoint-150", + "epoch": 1.5800922874093606, + "eval_steps": 50, + "global_step": 150, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01054713249835201, + "grad_norm": 11.254260063171387, + "learning_rate": 0.0, + "loss": 1.2856202125549316, + "step": 1 + }, + { + "epoch": 0.02109426499670402, + "grad_norm": 9.963000297546387, + "learning_rate": 6.666666666666667e-07, + "loss": 1.2529420852661133, + "step": 2 + }, + { + "epoch": 0.03164139749505603, + "grad_norm": 9.828377723693848, + "learning_rate": 1.3333333333333334e-06, + "loss": 1.1177468299865723, + "step": 3 + }, + { + "epoch": 0.04218852999340804, + "grad_norm": 8.927521705627441, + "learning_rate": 2.0000000000000003e-06, + "loss": 1.160679578781128, + "step": 4 + }, + { + "epoch": 0.05273566249176005, + "grad_norm": 9.450704574584961, + "learning_rate": 2.666666666666667e-06, + "loss": 1.1996268033981323, + "step": 5 + }, + { + "epoch": 0.06328279499011207, + "grad_norm": 7.224189758300781, + "learning_rate": 3.3333333333333333e-06, + "loss": 1.2939574718475342, + "step": 6 + }, + { + "epoch": 0.07382992748846408, + "grad_norm": 6.68184232711792, + "learning_rate": 4.000000000000001e-06, + "loss": 1.202090859413147, + "step": 7 + }, + { + "epoch": 0.08437705998681608, + "grad_norm": 5.252114772796631, + "learning_rate": 4.666666666666667e-06, + "loss": 1.0647697448730469, + "step": 8 + }, + { + "epoch": 0.09492419248516809, + "grad_norm": 4.770995140075684, + "learning_rate": 5.333333333333334e-06, + "loss": 0.811023473739624, + "step": 9 + }, + { + "epoch": 0.1054713249835201, + "grad_norm": 5.881967067718506, + "learning_rate": 6e-06, + "loss": 1.0770221948623657, + "step": 10 + }, + { + "epoch": 0.11601845748187212, + "grad_norm": 4.145230293273926, + "learning_rate": 6.666666666666667e-06, + "loss": 0.6524759531021118, + "step": 11 + }, + { + "epoch": 0.12656558998022413, + "grad_norm": 4.487707138061523, + "learning_rate": 7.333333333333333e-06, + "loss": 0.8969423770904541, + "step": 12 + }, + { + "epoch": 0.13711272247857614, + "grad_norm": 4.725864410400391, + "learning_rate": 8.000000000000001e-06, + "loss": 0.893510103225708, + "step": 13 + }, + { + "epoch": 0.14765985497692816, + "grad_norm": 5.280237674713135, + "learning_rate": 8.666666666666668e-06, + "loss": 1.3190690279006958, + "step": 14 + }, + { + "epoch": 0.15820698747528017, + "grad_norm": 4.238594055175781, + "learning_rate": 9.333333333333334e-06, + "loss": 0.8921085000038147, + "step": 15 + }, + { + "epoch": 0.16875411997363216, + "grad_norm": 4.737040042877197, + "learning_rate": 1e-05, + "loss": 1.1678017377853394, + "step": 16 + }, + { + "epoch": 0.17930125247198417, + "grad_norm": 3.861323356628418, + "learning_rate": 1.0666666666666667e-05, + "loss": 0.8146582841873169, + "step": 17 + }, + { + "epoch": 0.18984838497033618, + "grad_norm": 3.594571113586426, + "learning_rate": 1.1333333333333334e-05, + "loss": 0.6457481384277344, + "step": 18 + }, + { + "epoch": 0.2003955174686882, + "grad_norm": 5.211328983306885, + "learning_rate": 1.2e-05, + "loss": 1.250954031944275, + "step": 19 + }, + { + "epoch": 0.2109426499670402, + "grad_norm": 4.465060234069824, + "learning_rate": 1.2666666666666667e-05, + "loss": 1.146741271018982, + "step": 20 + }, + { + "epoch": 0.22148978246539222, + "grad_norm": 4.283473491668701, + "learning_rate": 1.3333333333333333e-05, + "loss": 1.0449175834655762, + "step": 21 + }, + { + "epoch": 0.23203691496374423, + "grad_norm": 4.074254512786865, + "learning_rate": 1.4e-05, + "loss": 1.132203459739685, + "step": 22 + }, + { + "epoch": 0.24258404746209625, + "grad_norm": 3.665837526321411, + "learning_rate": 1.4666666666666666e-05, + "loss": 0.7999849915504456, + "step": 23 + }, + { + "epoch": 0.25313117996044826, + "grad_norm": 3.750049352645874, + "learning_rate": 1.5333333333333334e-05, + "loss": 0.9189183115959167, + "step": 24 + }, + { + "epoch": 0.26367831245880025, + "grad_norm": 3.9863851070404053, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.9486135244369507, + "step": 25 + }, + { + "epoch": 0.2742254449571523, + "grad_norm": 3.5417375564575195, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.7232263088226318, + "step": 26 + }, + { + "epoch": 0.28477257745550427, + "grad_norm": 2.7772789001464844, + "learning_rate": 1.7333333333333336e-05, + "loss": 0.5559145212173462, + "step": 27 + }, + { + "epoch": 0.2953197099538563, + "grad_norm": 3.2987618446350098, + "learning_rate": 1.8e-05, + "loss": 0.7673175930976868, + "step": 28 + }, + { + "epoch": 0.3058668424522083, + "grad_norm": 4.202295303344727, + "learning_rate": 1.866666666666667e-05, + "loss": 0.9726297855377197, + "step": 29 + }, + { + "epoch": 0.31641397495056034, + "grad_norm": 3.3992533683776855, + "learning_rate": 1.9333333333333333e-05, + "loss": 0.8889808058738708, + "step": 30 + }, + { + "epoch": 0.3269611074489123, + "grad_norm": 3.55267596244812, + "learning_rate": 2e-05, + "loss": 0.7761465907096863, + "step": 31 + }, + { + "epoch": 0.3375082399472643, + "grad_norm": 3.9312660694122314, + "learning_rate": 1.9998072404820648e-05, + "loss": 1.0842286348342896, + "step": 32 + }, + { + "epoch": 0.34805537244561635, + "grad_norm": 3.166452407836914, + "learning_rate": 1.9992290362407232e-05, + "loss": 0.8452663421630859, + "step": 33 + }, + { + "epoch": 0.35860250494396834, + "grad_norm": 3.7628989219665527, + "learning_rate": 1.998265610184716e-05, + "loss": 0.9143624305725098, + "step": 34 + }, + { + "epoch": 0.3691496374423204, + "grad_norm": 3.930147409439087, + "learning_rate": 1.9969173337331283e-05, + "loss": 0.937317967414856, + "step": 35 + }, + { + "epoch": 0.37969676994067236, + "grad_norm": 3.9495177268981934, + "learning_rate": 1.995184726672197e-05, + "loss": 0.9962741732597351, + "step": 36 + }, + { + "epoch": 0.3902439024390244, + "grad_norm": 3.237985849380493, + "learning_rate": 1.9930684569549265e-05, + "loss": 0.8508425354957581, + "step": 37 + }, + { + "epoch": 0.4007910349373764, + "grad_norm": 3.5464437007904053, + "learning_rate": 1.990569340443577e-05, + "loss": 1.0232917070388794, + "step": 38 + }, + { + "epoch": 0.41133816743572843, + "grad_norm": 2.8413944244384766, + "learning_rate": 1.9876883405951378e-05, + "loss": 0.7485581040382385, + "step": 39 + }, + { + "epoch": 0.4218852999340804, + "grad_norm": 3.03243088722229, + "learning_rate": 1.9844265680898917e-05, + "loss": 0.8981056809425354, + "step": 40 + }, + { + "epoch": 0.43243243243243246, + "grad_norm": 2.713469982147217, + "learning_rate": 1.9807852804032306e-05, + "loss": 0.7302929162979126, + "step": 41 + }, + { + "epoch": 0.44297956493078444, + "grad_norm": 2.5833826065063477, + "learning_rate": 1.9767658813208725e-05, + "loss": 0.8823336958885193, + "step": 42 + }, + { + "epoch": 0.4535266974291364, + "grad_norm": 3.5490224361419678, + "learning_rate": 1.9723699203976768e-05, + "loss": 0.9176093935966492, + "step": 43 + }, + { + "epoch": 0.46407382992748847, + "grad_norm": 2.9480419158935547, + "learning_rate": 1.96759909236026e-05, + "loss": 0.7887524962425232, + "step": 44 + }, + { + "epoch": 0.47462096242584045, + "grad_norm": 2.722837448120117, + "learning_rate": 1.9624552364536472e-05, + "loss": 0.7906816601753235, + "step": 45 + }, + { + "epoch": 0.4851680949241925, + "grad_norm": 2.400080919265747, + "learning_rate": 1.956940335732209e-05, + "loss": 0.5717998147010803, + "step": 46 + }, + { + "epoch": 0.4957152274225445, + "grad_norm": 3.238320827484131, + "learning_rate": 1.9510565162951538e-05, + "loss": 0.8687648773193359, + "step": 47 + }, + { + "epoch": 0.5062623599208965, + "grad_norm": 3.124903678894043, + "learning_rate": 1.944806046466878e-05, + "loss": 0.9546631574630737, + "step": 48 + }, + { + "epoch": 0.5168094924192486, + "grad_norm": 3.41595458984375, + "learning_rate": 1.9381913359224844e-05, + "loss": 0.7677831649780273, + "step": 49 + }, + { + "epoch": 0.5273566249176005, + "grad_norm": 2.969999313354492, + "learning_rate": 1.9312149347588035e-05, + "loss": 0.8908669352531433, + "step": 50 + }, + { + "epoch": 0.5273566249176005, + "eval_loss": 0.7529963850975037, + "eval_runtime": 28.3177, + "eval_samples_per_second": 5.05, + "eval_steps_per_second": 1.271, + "step": 50 + }, + { + "epoch": 0.5379037574159525, + "grad_norm": 2.576383113861084, + "learning_rate": 1.9238795325112867e-05, + "loss": 0.7306627631187439, + "step": 51 + }, + { + "epoch": 0.5484508899143046, + "grad_norm": 2.5779168605804443, + "learning_rate": 1.916187957117136e-05, + "loss": 0.6502229571342468, + "step": 52 + }, + { + "epoch": 0.5589980224126566, + "grad_norm": 3.7034096717834473, + "learning_rate": 1.9081431738250815e-05, + "loss": 1.0519064664840698, + "step": 53 + }, + { + "epoch": 0.5695451549110085, + "grad_norm": 3.7297651767730713, + "learning_rate": 1.8997482840522218e-05, + "loss": 0.9438788890838623, + "step": 54 + }, + { + "epoch": 0.5800922874093606, + "grad_norm": 2.983699083328247, + "learning_rate": 1.891006524188368e-05, + "loss": 0.810992419719696, + "step": 55 + }, + { + "epoch": 0.5906394199077126, + "grad_norm": 2.869706869125366, + "learning_rate": 1.881921264348355e-05, + "loss": 0.681011438369751, + "step": 56 + }, + { + "epoch": 0.6011865524060646, + "grad_norm": 2.8603360652923584, + "learning_rate": 1.8724960070727974e-05, + "loss": 0.8332223892211914, + "step": 57 + }, + { + "epoch": 0.6117336849044166, + "grad_norm": 2.766416072845459, + "learning_rate": 1.862734385977792e-05, + "loss": 0.8047686815261841, + "step": 58 + }, + { + "epoch": 0.6222808174027686, + "grad_norm": 2.664792776107788, + "learning_rate": 1.8526401643540924e-05, + "loss": 0.6811239719390869, + "step": 59 + }, + { + "epoch": 0.6328279499011207, + "grad_norm": 2.787421226501465, + "learning_rate": 1.8422172337162865e-05, + "loss": 0.9071734547615051, + "step": 60 + }, + { + "epoch": 0.6433750823994726, + "grad_norm": 2.5916903018951416, + "learning_rate": 1.8314696123025456e-05, + "loss": 0.778200089931488, + "step": 61 + }, + { + "epoch": 0.6539222148978246, + "grad_norm": 2.5858092308044434, + "learning_rate": 1.8204014435255136e-05, + "loss": 0.7620019912719727, + "step": 62 + }, + { + "epoch": 0.6644693473961767, + "grad_norm": 3.226935386657715, + "learning_rate": 1.8090169943749477e-05, + "loss": 1.0030763149261475, + "step": 63 + }, + { + "epoch": 0.6750164798945286, + "grad_norm": 2.46641206741333, + "learning_rate": 1.797320653772707e-05, + "loss": 0.6492084860801697, + "step": 64 + }, + { + "epoch": 0.6855636123928807, + "grad_norm": 2.243069648742676, + "learning_rate": 1.785316930880745e-05, + "loss": 0.6458125114440918, + "step": 65 + }, + { + "epoch": 0.6961107448912327, + "grad_norm": 2.7569828033447266, + "learning_rate": 1.773010453362737e-05, + "loss": 0.8067665100097656, + "step": 66 + }, + { + "epoch": 0.7066578773895847, + "grad_norm": 3.01151967048645, + "learning_rate": 1.7604059656000313e-05, + "loss": 0.8096626996994019, + "step": 67 + }, + { + "epoch": 0.7172050098879367, + "grad_norm": 2.5362462997436523, + "learning_rate": 1.747508326862597e-05, + "loss": 0.5855382084846497, + "step": 68 + }, + { + "epoch": 0.7277521423862887, + "grad_norm": 3.2442595958709717, + "learning_rate": 1.7343225094356857e-05, + "loss": 1.0010881423950195, + "step": 69 + }, + { + "epoch": 0.7382992748846408, + "grad_norm": 2.9444711208343506, + "learning_rate": 1.720853596702919e-05, + "loss": 0.9237573146820068, + "step": 70 + }, + { + "epoch": 0.7488464073829928, + "grad_norm": 2.2847354412078857, + "learning_rate": 1.7071067811865477e-05, + "loss": 0.6892184615135193, + "step": 71 + }, + { + "epoch": 0.7593935398813447, + "grad_norm": 2.72588849067688, + "learning_rate": 1.6930873625456362e-05, + "loss": 0.7187391519546509, + "step": 72 + }, + { + "epoch": 0.7699406723796968, + "grad_norm": 2.462245225906372, + "learning_rate": 1.678800745532942e-05, + "loss": 0.6943771839141846, + "step": 73 + }, + { + "epoch": 0.7804878048780488, + "grad_norm": 2.880293846130371, + "learning_rate": 1.664252437911282e-05, + "loss": 0.7184453010559082, + "step": 74 + }, + { + "epoch": 0.7910349373764007, + "grad_norm": 3.225119113922119, + "learning_rate": 1.6494480483301836e-05, + "loss": 0.9926764369010925, + "step": 75 + }, + { + "epoch": 0.8015820698747528, + "grad_norm": 2.788250207901001, + "learning_rate": 1.6343932841636455e-05, + "loss": 0.8473926186561584, + "step": 76 + }, + { + "epoch": 0.8121292023731048, + "grad_norm": 2.1925415992736816, + "learning_rate": 1.6190939493098344e-05, + "loss": 0.5790418982505798, + "step": 77 + }, + { + "epoch": 0.8226763348714569, + "grad_norm": 2.4412641525268555, + "learning_rate": 1.6035559419535714e-05, + "loss": 0.677460789680481, + "step": 78 + }, + { + "epoch": 0.8332234673698088, + "grad_norm": 2.478698968887329, + "learning_rate": 1.5877852522924733e-05, + "loss": 0.8228996992111206, + "step": 79 + }, + { + "epoch": 0.8437705998681608, + "grad_norm": 2.5369465351104736, + "learning_rate": 1.5717879602276123e-05, + "loss": 0.7633792757987976, + "step": 80 + }, + { + "epoch": 0.8543177323665129, + "grad_norm": 2.9730212688446045, + "learning_rate": 1.5555702330196024e-05, + "loss": 1.0669782161712646, + "step": 81 + }, + { + "epoch": 0.8648648648648649, + "grad_norm": 2.1946942806243896, + "learning_rate": 1.5391383229110005e-05, + "loss": 0.6726582646369934, + "step": 82 + }, + { + "epoch": 0.8754119973632168, + "grad_norm": 2.4897115230560303, + "learning_rate": 1.5224985647159489e-05, + "loss": 0.7702843546867371, + "step": 83 + }, + { + "epoch": 0.8859591298615689, + "grad_norm": 2.055375337600708, + "learning_rate": 1.5056573733779848e-05, + "loss": 0.49126309156417847, + "step": 84 + }, + { + "epoch": 0.8965062623599209, + "grad_norm": 2.346480131149292, + "learning_rate": 1.4886212414969551e-05, + "loss": 0.7343087196350098, + "step": 85 + }, + { + "epoch": 0.9070533948582729, + "grad_norm": 2.2193572521209717, + "learning_rate": 1.4713967368259981e-05, + "loss": 0.5628997683525085, + "step": 86 + }, + { + "epoch": 0.9176005273566249, + "grad_norm": 2.818474292755127, + "learning_rate": 1.4539904997395468e-05, + "loss": 0.9049317240715027, + "step": 87 + }, + { + "epoch": 0.9281476598549769, + "grad_norm": 2.0409064292907715, + "learning_rate": 1.436409240673342e-05, + "loss": 0.609929621219635, + "step": 88 + }, + { + "epoch": 0.938694792353329, + "grad_norm": 2.5648860931396484, + "learning_rate": 1.4186597375374283e-05, + "loss": 0.7217040657997131, + "step": 89 + }, + { + "epoch": 0.9492419248516809, + "grad_norm": 2.435124158859253, + "learning_rate": 1.4007488331031409e-05, + "loss": 0.6683085560798645, + "step": 90 + }, + { + "epoch": 0.959789057350033, + "grad_norm": 2.642301082611084, + "learning_rate": 1.3826834323650899e-05, + "loss": 0.7930483222007751, + "step": 91 + }, + { + "epoch": 0.970336189848385, + "grad_norm": 2.8316915035247803, + "learning_rate": 1.3644704998791501e-05, + "loss": 0.8307502269744873, + "step": 92 + }, + { + "epoch": 0.980883322346737, + "grad_norm": 2.585615873336792, + "learning_rate": 1.346117057077493e-05, + "loss": 0.809612512588501, + "step": 93 + }, + { + "epoch": 0.991430454845089, + "grad_norm": 2.4871203899383545, + "learning_rate": 1.3276301795616937e-05, + "loss": 0.7022823095321655, + "step": 94 + }, + { + "epoch": 1.0, + "grad_norm": 2.2630867958068848, + "learning_rate": 1.3090169943749475e-05, + "loss": 0.540604293346405, + "step": 95 + }, + { + "epoch": 1.010547132498352, + "grad_norm": 1.8352792263031006, + "learning_rate": 1.2902846772544625e-05, + "loss": 0.47405001521110535, + "step": 96 + }, + { + "epoch": 1.021094264996704, + "grad_norm": 2.3510472774505615, + "learning_rate": 1.2714404498650743e-05, + "loss": 0.7485337257385254, + "step": 97 + }, + { + "epoch": 1.031641397495056, + "grad_norm": 2.6154630184173584, + "learning_rate": 1.252491577015158e-05, + "loss": 0.8424244523048401, + "step": 98 + }, + { + "epoch": 1.042188529993408, + "grad_norm": 2.235624074935913, + "learning_rate": 1.2334453638559057e-05, + "loss": 0.5035255551338196, + "step": 99 + }, + { + "epoch": 1.05273566249176, + "grad_norm": 2.5067241191864014, + "learning_rate": 1.2143091530650508e-05, + "loss": 0.839946985244751, + "step": 100 + }, + { + "epoch": 1.05273566249176, + "eval_loss": 0.69390469789505, + "eval_runtime": 28.1554, + "eval_samples_per_second": 5.079, + "eval_steps_per_second": 1.279, + "step": 100 + }, + { + "epoch": 1.063282794990112, + "grad_norm": 2.4152822494506836, + "learning_rate": 1.1950903220161286e-05, + "loss": 0.6019119024276733, + "step": 101 + }, + { + "epoch": 1.0738299274884642, + "grad_norm": 3.1178929805755615, + "learning_rate": 1.1757962799343548e-05, + "loss": 0.8134055733680725, + "step": 102 + }, + { + "epoch": 1.084377059986816, + "grad_norm": 2.630704402923584, + "learning_rate": 1.156434465040231e-05, + "loss": 0.7413411736488342, + "step": 103 + }, + { + "epoch": 1.094924192485168, + "grad_norm": 2.8832743167877197, + "learning_rate": 1.1370123416819683e-05, + "loss": 0.8591673374176025, + "step": 104 + }, + { + "epoch": 1.1054713249835202, + "grad_norm": 2.6520588397979736, + "learning_rate": 1.1175373974578378e-05, + "loss": 0.7692611813545227, + "step": 105 + }, + { + "epoch": 1.1160184574818721, + "grad_norm": 2.3777475357055664, + "learning_rate": 1.098017140329561e-05, + "loss": 0.644071102142334, + "step": 106 + }, + { + "epoch": 1.126565589980224, + "grad_norm": 2.3438470363616943, + "learning_rate": 1.0784590957278452e-05, + "loss": 0.6566027998924255, + "step": 107 + }, + { + "epoch": 1.1371127224785762, + "grad_norm": 2.494967222213745, + "learning_rate": 1.058870803651189e-05, + "loss": 0.6349762678146362, + "step": 108 + }, + { + "epoch": 1.1476598549769281, + "grad_norm": 2.557257652282715, + "learning_rate": 1.0392598157590687e-05, + "loss": 0.7407976388931274, + "step": 109 + }, + { + "epoch": 1.15820698747528, + "grad_norm": 2.54610013961792, + "learning_rate": 1.0196336924606282e-05, + "loss": 0.6565055251121521, + "step": 110 + }, + { + "epoch": 1.1687541199736322, + "grad_norm": 2.7403500080108643, + "learning_rate": 1e-05, + "loss": 0.9395892024040222, + "step": 111 + }, + { + "epoch": 1.1793012524719841, + "grad_norm": 2.6010167598724365, + "learning_rate": 9.80366307539372e-06, + "loss": 0.7564979791641235, + "step": 112 + }, + { + "epoch": 1.189848384970336, + "grad_norm": 2.5348997116088867, + "learning_rate": 9.607401842409318e-06, + "loss": 0.671207070350647, + "step": 113 + }, + { + "epoch": 1.2003955174686882, + "grad_norm": 2.2326834201812744, + "learning_rate": 9.41129196348811e-06, + "loss": 0.4958484470844269, + "step": 114 + }, + { + "epoch": 1.2109426499670402, + "grad_norm": 2.7414183616638184, + "learning_rate": 9.215409042721553e-06, + "loss": 0.7510374784469604, + "step": 115 + }, + { + "epoch": 1.2214897824653923, + "grad_norm": 2.51889705657959, + "learning_rate": 9.019828596704394e-06, + "loss": 0.6310056447982788, + "step": 116 + }, + { + "epoch": 1.2320369149637442, + "grad_norm": 2.1737775802612305, + "learning_rate": 8.824626025421625e-06, + "loss": 0.5721197128295898, + "step": 117 + }, + { + "epoch": 1.2425840474620962, + "grad_norm": 2.6601269245147705, + "learning_rate": 8.629876583180322e-06, + "loss": 0.6723504662513733, + "step": 118 + }, + { + "epoch": 1.2531311799604483, + "grad_norm": 3.153501033782959, + "learning_rate": 8.43565534959769e-06, + "loss": 0.8398867845535278, + "step": 119 + }, + { + "epoch": 1.2636783124588002, + "grad_norm": 2.913832664489746, + "learning_rate": 8.242037200656455e-06, + "loss": 0.7067492008209229, + "step": 120 + }, + { + "epoch": 1.2742254449571524, + "grad_norm": 3.4029502868652344, + "learning_rate": 8.04909677983872e-06, + "loss": 0.9664009809494019, + "step": 121 + }, + { + "epoch": 1.2847725774555043, + "grad_norm": 2.364175319671631, + "learning_rate": 7.856908469349495e-06, + "loss": 0.5740135312080383, + "step": 122 + }, + { + "epoch": 1.2953197099538563, + "grad_norm": 2.6324050426483154, + "learning_rate": 7.66554636144095e-06, + "loss": 0.6533321738243103, + "step": 123 + }, + { + "epoch": 1.3058668424522084, + "grad_norm": 2.7875795364379883, + "learning_rate": 7.4750842298484205e-06, + "loss": 0.7847114205360413, + "step": 124 + }, + { + "epoch": 1.3164139749505603, + "grad_norm": 2.482607126235962, + "learning_rate": 7.285595501349259e-06, + "loss": 0.6538381576538086, + "step": 125 + }, + { + "epoch": 1.3269611074489123, + "grad_norm": 2.798854351043701, + "learning_rate": 7.097153227455379e-06, + "loss": 0.7187844514846802, + "step": 126 + }, + { + "epoch": 1.3375082399472644, + "grad_norm": 3.029470920562744, + "learning_rate": 6.909830056250527e-06, + "loss": 0.859753429889679, + "step": 127 + }, + { + "epoch": 1.3480553724456164, + "grad_norm": 2.3846898078918457, + "learning_rate": 6.723698204383067e-06, + "loss": 0.6410857439041138, + "step": 128 + }, + { + "epoch": 1.3586025049439683, + "grad_norm": 2.828608751296997, + "learning_rate": 6.538829429225068e-06, + "loss": 0.7521955966949463, + "step": 129 + }, + { + "epoch": 1.3691496374423204, + "grad_norm": 2.1056339740753174, + "learning_rate": 6.355295001208504e-06, + "loss": 0.49943816661834717, + "step": 130 + }, + { + "epoch": 1.3796967699406724, + "grad_norm": 2.9909141063690186, + "learning_rate": 6.173165676349103e-06, + "loss": 0.6608769297599792, + "step": 131 + }, + { + "epoch": 1.3902439024390243, + "grad_norm": 2.3114278316497803, + "learning_rate": 5.9925116689685925e-06, + "loss": 0.5453819632530212, + "step": 132 + }, + { + "epoch": 1.4007910349373764, + "grad_norm": 2.200542688369751, + "learning_rate": 5.813402624625722e-06, + "loss": 0.4996246099472046, + "step": 133 + }, + { + "epoch": 1.4113381674357284, + "grad_norm": 2.2516205310821533, + "learning_rate": 5.635907593266578e-06, + "loss": 0.63039630651474, + "step": 134 + }, + { + "epoch": 1.4218852999340803, + "grad_norm": 2.5531094074249268, + "learning_rate": 5.460095002604533e-06, + "loss": 0.65385502576828, + "step": 135 + }, + { + "epoch": 1.4324324324324325, + "grad_norm": 2.369076728820801, + "learning_rate": 5.286032631740023e-06, + "loss": 0.5936092138290405, + "step": 136 + }, + { + "epoch": 1.4429795649307844, + "grad_norm": 2.984978199005127, + "learning_rate": 5.1137875850304545e-06, + "loss": 0.7236067652702332, + "step": 137 + }, + { + "epoch": 1.4535266974291363, + "grad_norm": 2.5663387775421143, + "learning_rate": 4.943426266220156e-06, + "loss": 0.6463375091552734, + "step": 138 + }, + { + "epoch": 1.4640738299274885, + "grad_norm": 2.425461769104004, + "learning_rate": 4.775014352840512e-06, + "loss": 0.6277223825454712, + "step": 139 + }, + { + "epoch": 1.4746209624258404, + "grad_norm": 2.8183038234710693, + "learning_rate": 4.608616770889998e-06, + "loss": 0.8176087141036987, + "step": 140 + }, + { + "epoch": 1.4851680949241926, + "grad_norm": 2.203562021255493, + "learning_rate": 4.444297669803981e-06, + "loss": 0.4891142249107361, + "step": 141 + }, + { + "epoch": 1.4957152274225445, + "grad_norm": 2.412172794342041, + "learning_rate": 4.282120397723879e-06, + "loss": 0.735496997833252, + "step": 142 + }, + { + "epoch": 1.5062623599208966, + "grad_norm": 2.850745916366577, + "learning_rate": 4.12214747707527e-06, + "loss": 0.736525297164917, + "step": 143 + }, + { + "epoch": 1.5168094924192486, + "grad_norm": 2.6273257732391357, + "learning_rate": 3.964440580464286e-06, + "loss": 0.629302978515625, + "step": 144 + }, + { + "epoch": 1.5273566249176005, + "grad_norm": 2.045539379119873, + "learning_rate": 3.8090605069016596e-06, + "loss": 0.5002003312110901, + "step": 145 + }, + { + "epoch": 1.5379037574159526, + "grad_norm": 2.5290746688842773, + "learning_rate": 3.6560671583635467e-06, + "loss": 0.604846715927124, + "step": 146 + }, + { + "epoch": 1.5484508899143046, + "grad_norm": 2.6563849449157715, + "learning_rate": 3.505519516698165e-06, + "loss": 0.7351030111312866, + "step": 147 + }, + { + "epoch": 1.5589980224126565, + "grad_norm": 2.430051326751709, + "learning_rate": 3.3574756208871862e-06, + "loss": 0.49856996536254883, + "step": 148 + }, + { + "epoch": 1.5695451549110087, + "grad_norm": 2.4349019527435303, + "learning_rate": 3.2119925446705824e-06, + "loss": 0.6530035138130188, + "step": 149 + }, + { + "epoch": 1.5800922874093606, + "grad_norm": 2.302081823348999, + "learning_rate": 3.069126374543643e-06, + "loss": 0.5958735346794128, + "step": 150 + }, + { + "epoch": 1.5800922874093606, + "eval_loss": 0.6550542712211609, + "eval_runtime": 27.788, + "eval_samples_per_second": 5.146, + "eval_steps_per_second": 1.296, + "step": 150 + } + ], + "logging_steps": 1, + "max_steps": 190, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.316898167598182e+16, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/lora_adapter/checkpoint-150/training_args.bin b/lora_adapter/checkpoint-150/training_args.bin new file mode 100644 index 0000000..9b1b43f --- /dev/null +++ b/lora_adapter/checkpoint-150/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a35a78b99e3a3a7f629b461e552407e5f338f437d3bf7df51fb87ffb0945bec7 +size 5713 diff --git a/lora_adapter/checkpoint-190/README.md b/lora_adapter/checkpoint-190/README.md new file mode 100644 index 0000000..a4c765d --- /dev/null +++ b/lora_adapter/checkpoint-190/README.md @@ -0,0 +1,210 @@ +--- +base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +- lora +- sft +- transformers +- trl +- unsloth +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/lora_adapter/checkpoint-190/adapter_config.json b/lora_adapter/checkpoint-190/adapter_config.json new file mode 100644 index 0000000..cdc21db --- /dev/null +++ b/lora_adapter/checkpoint-190/adapter_config.json @@ -0,0 +1,52 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": { + "base_model_class": "Qwen2ForCausalLM", + "parent_library": "transformers.models.qwen2.modeling_qwen2", + "unsloth_fixed": true + }, + "base_model_name_or_path": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "o_proj", + "k_proj", + "q_proj", + "gate_proj", + "v_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": true +} \ No newline at end of file diff --git a/lora_adapter/checkpoint-190/adapter_model.safetensors b/lora_adapter/checkpoint-190/adapter_model.safetensors new file mode 100644 index 0000000..af7e749 --- /dev/null +++ b/lora_adapter/checkpoint-190/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46babf050836e50ce540a66fa0d2fd0b2c7914b8b902b8e46ccd48685d72c6a0 +size 590925768 diff --git a/lora_adapter/checkpoint-190/chat_template.jinja b/lora_adapter/checkpoint-190/chat_template.jinja new file mode 100644 index 0000000..642e597 --- /dev/null +++ b/lora_adapter/checkpoint-190/chat_template.jinja @@ -0,0 +1,53 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/lora_adapter/checkpoint-190/optimizer.pt b/lora_adapter/checkpoint-190/optimizer.pt new file mode 100644 index 0000000..fd5c72d --- /dev/null +++ b/lora_adapter/checkpoint-190/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0004abefaba596f28cb85c71ee507e076912b7052bae77538bce615e30ff24e7 +size 300517573 diff --git a/lora_adapter/checkpoint-190/rng_state.pth b/lora_adapter/checkpoint-190/rng_state.pth new file mode 100644 index 0000000..3ab6836 --- /dev/null +++ b/lora_adapter/checkpoint-190/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01f9a0f7843a37be87edd23f4e88aa93b38b95cc2c07503eeb1cf2e4632453a2 +size 14645 diff --git a/lora_adapter/checkpoint-190/scheduler.pt b/lora_adapter/checkpoint-190/scheduler.pt new file mode 100644 index 0000000..d09e9fa --- /dev/null +++ b/lora_adapter/checkpoint-190/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e1d80ff2ccd5ff39c18589bd5d49a23be49f50466127bf591653f2801a5a856 +size 1465 diff --git a/lora_adapter/checkpoint-190/tokenizer.json b/lora_adapter/checkpoint-190/tokenizer.json new file mode 100644 index 0000000..5340d81 --- /dev/null +++ b/lora_adapter/checkpoint-190/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd5948af71b4f56cf697f7580814c7ce8b80595ef985544efcacf716126a2e31 +size 11422356 diff --git a/lora_adapter/checkpoint-190/tokenizer_config.json b/lora_adapter/checkpoint-190/tokenizer_config.json new file mode 100644 index 0000000..d3dac53 --- /dev/null +++ b/lora_adapter/checkpoint-190/tokenizer_config.json @@ -0,0 +1,201 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 32768, + "pad_token": "<|PAD_TOKEN|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151657": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151658": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151665": { + "content": "<|PAD_TOKEN|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + } +} diff --git a/lora_adapter/checkpoint-190/trainer_state.json b/lora_adapter/checkpoint-190/trainer_state.json new file mode 100644 index 0000000..b16bc4f --- /dev/null +++ b/lora_adapter/checkpoint-190/trainer_state.json @@ -0,0 +1,1396 @@ +{ + "best_global_step": 190, + "best_metric": 0.6482135057449341, + "best_model_checkpoint": "./qwen_unified_finetuned_v2/checkpoint-190", + "epoch": 2.0, + "eval_steps": 50, + "global_step": 190, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.01054713249835201, + "grad_norm": 11.254260063171387, + "learning_rate": 0.0, + "loss": 1.2856202125549316, + "step": 1 + }, + { + "epoch": 0.02109426499670402, + "grad_norm": 9.963000297546387, + "learning_rate": 6.666666666666667e-07, + "loss": 1.2529420852661133, + "step": 2 + }, + { + "epoch": 0.03164139749505603, + "grad_norm": 9.828377723693848, + "learning_rate": 1.3333333333333334e-06, + "loss": 1.1177468299865723, + "step": 3 + }, + { + "epoch": 0.04218852999340804, + "grad_norm": 8.927521705627441, + "learning_rate": 2.0000000000000003e-06, + "loss": 1.160679578781128, + "step": 4 + }, + { + "epoch": 0.05273566249176005, + "grad_norm": 9.450704574584961, + "learning_rate": 2.666666666666667e-06, + "loss": 1.1996268033981323, + "step": 5 + }, + { + "epoch": 0.06328279499011207, + "grad_norm": 7.224189758300781, + "learning_rate": 3.3333333333333333e-06, + "loss": 1.2939574718475342, + "step": 6 + }, + { + "epoch": 0.07382992748846408, + "grad_norm": 6.68184232711792, + "learning_rate": 4.000000000000001e-06, + "loss": 1.202090859413147, + "step": 7 + }, + { + "epoch": 0.08437705998681608, + "grad_norm": 5.252114772796631, + "learning_rate": 4.666666666666667e-06, + "loss": 1.0647697448730469, + "step": 8 + }, + { + "epoch": 0.09492419248516809, + "grad_norm": 4.770995140075684, + "learning_rate": 5.333333333333334e-06, + "loss": 0.811023473739624, + "step": 9 + }, + { + "epoch": 0.1054713249835201, + "grad_norm": 5.881967067718506, + "learning_rate": 6e-06, + "loss": 1.0770221948623657, + "step": 10 + }, + { + "epoch": 0.11601845748187212, + "grad_norm": 4.145230293273926, + "learning_rate": 6.666666666666667e-06, + "loss": 0.6524759531021118, + "step": 11 + }, + { + "epoch": 0.12656558998022413, + "grad_norm": 4.487707138061523, + "learning_rate": 7.333333333333333e-06, + "loss": 0.8969423770904541, + "step": 12 + }, + { + "epoch": 0.13711272247857614, + "grad_norm": 4.725864410400391, + "learning_rate": 8.000000000000001e-06, + "loss": 0.893510103225708, + "step": 13 + }, + { + "epoch": 0.14765985497692816, + "grad_norm": 5.280237674713135, + "learning_rate": 8.666666666666668e-06, + "loss": 1.3190690279006958, + "step": 14 + }, + { + "epoch": 0.15820698747528017, + "grad_norm": 4.238594055175781, + "learning_rate": 9.333333333333334e-06, + "loss": 0.8921085000038147, + "step": 15 + }, + { + "epoch": 0.16875411997363216, + "grad_norm": 4.737040042877197, + "learning_rate": 1e-05, + "loss": 1.1678017377853394, + "step": 16 + }, + { + "epoch": 0.17930125247198417, + "grad_norm": 3.861323356628418, + "learning_rate": 1.0666666666666667e-05, + "loss": 0.8146582841873169, + "step": 17 + }, + { + "epoch": 0.18984838497033618, + "grad_norm": 3.594571113586426, + "learning_rate": 1.1333333333333334e-05, + "loss": 0.6457481384277344, + "step": 18 + }, + { + "epoch": 0.2003955174686882, + "grad_norm": 5.211328983306885, + "learning_rate": 1.2e-05, + "loss": 1.250954031944275, + "step": 19 + }, + { + "epoch": 0.2109426499670402, + "grad_norm": 4.465060234069824, + "learning_rate": 1.2666666666666667e-05, + "loss": 1.146741271018982, + "step": 20 + }, + { + "epoch": 0.22148978246539222, + "grad_norm": 4.283473491668701, + "learning_rate": 1.3333333333333333e-05, + "loss": 1.0449175834655762, + "step": 21 + }, + { + "epoch": 0.23203691496374423, + "grad_norm": 4.074254512786865, + "learning_rate": 1.4e-05, + "loss": 1.132203459739685, + "step": 22 + }, + { + "epoch": 0.24258404746209625, + "grad_norm": 3.665837526321411, + "learning_rate": 1.4666666666666666e-05, + "loss": 0.7999849915504456, + "step": 23 + }, + { + "epoch": 0.25313117996044826, + "grad_norm": 3.750049352645874, + "learning_rate": 1.5333333333333334e-05, + "loss": 0.9189183115959167, + "step": 24 + }, + { + "epoch": 0.26367831245880025, + "grad_norm": 3.9863851070404053, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.9486135244369507, + "step": 25 + }, + { + "epoch": 0.2742254449571523, + "grad_norm": 3.5417375564575195, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.7232263088226318, + "step": 26 + }, + { + "epoch": 0.28477257745550427, + "grad_norm": 2.7772789001464844, + "learning_rate": 1.7333333333333336e-05, + "loss": 0.5559145212173462, + "step": 27 + }, + { + "epoch": 0.2953197099538563, + "grad_norm": 3.2987618446350098, + "learning_rate": 1.8e-05, + "loss": 0.7673175930976868, + "step": 28 + }, + { + "epoch": 0.3058668424522083, + "grad_norm": 4.202295303344727, + "learning_rate": 1.866666666666667e-05, + "loss": 0.9726297855377197, + "step": 29 + }, + { + "epoch": 0.31641397495056034, + "grad_norm": 3.3992533683776855, + "learning_rate": 1.9333333333333333e-05, + "loss": 0.8889808058738708, + "step": 30 + }, + { + "epoch": 0.3269611074489123, + "grad_norm": 3.55267596244812, + "learning_rate": 2e-05, + "loss": 0.7761465907096863, + "step": 31 + }, + { + "epoch": 0.3375082399472643, + "grad_norm": 3.9312660694122314, + "learning_rate": 1.9998072404820648e-05, + "loss": 1.0842286348342896, + "step": 32 + }, + { + "epoch": 0.34805537244561635, + "grad_norm": 3.166452407836914, + "learning_rate": 1.9992290362407232e-05, + "loss": 0.8452663421630859, + "step": 33 + }, + { + "epoch": 0.35860250494396834, + "grad_norm": 3.7628989219665527, + "learning_rate": 1.998265610184716e-05, + "loss": 0.9143624305725098, + "step": 34 + }, + { + "epoch": 0.3691496374423204, + "grad_norm": 3.930147409439087, + "learning_rate": 1.9969173337331283e-05, + "loss": 0.937317967414856, + "step": 35 + }, + { + "epoch": 0.37969676994067236, + "grad_norm": 3.9495177268981934, + "learning_rate": 1.995184726672197e-05, + "loss": 0.9962741732597351, + "step": 36 + }, + { + "epoch": 0.3902439024390244, + "grad_norm": 3.237985849380493, + "learning_rate": 1.9930684569549265e-05, + "loss": 0.8508425354957581, + "step": 37 + }, + { + "epoch": 0.4007910349373764, + "grad_norm": 3.5464437007904053, + "learning_rate": 1.990569340443577e-05, + "loss": 1.0232917070388794, + "step": 38 + }, + { + "epoch": 0.41133816743572843, + "grad_norm": 2.8413944244384766, + "learning_rate": 1.9876883405951378e-05, + "loss": 0.7485581040382385, + "step": 39 + }, + { + "epoch": 0.4218852999340804, + "grad_norm": 3.03243088722229, + "learning_rate": 1.9844265680898917e-05, + "loss": 0.8981056809425354, + "step": 40 + }, + { + "epoch": 0.43243243243243246, + "grad_norm": 2.713469982147217, + "learning_rate": 1.9807852804032306e-05, + "loss": 0.7302929162979126, + "step": 41 + }, + { + "epoch": 0.44297956493078444, + "grad_norm": 2.5833826065063477, + "learning_rate": 1.9767658813208725e-05, + "loss": 0.8823336958885193, + "step": 42 + }, + { + "epoch": 0.4535266974291364, + "grad_norm": 3.5490224361419678, + "learning_rate": 1.9723699203976768e-05, + "loss": 0.9176093935966492, + "step": 43 + }, + { + "epoch": 0.46407382992748847, + "grad_norm": 2.9480419158935547, + "learning_rate": 1.96759909236026e-05, + "loss": 0.7887524962425232, + "step": 44 + }, + { + "epoch": 0.47462096242584045, + "grad_norm": 2.722837448120117, + "learning_rate": 1.9624552364536472e-05, + "loss": 0.7906816601753235, + "step": 45 + }, + { + "epoch": 0.4851680949241925, + "grad_norm": 2.400080919265747, + "learning_rate": 1.956940335732209e-05, + "loss": 0.5717998147010803, + "step": 46 + }, + { + "epoch": 0.4957152274225445, + "grad_norm": 3.238320827484131, + "learning_rate": 1.9510565162951538e-05, + "loss": 0.8687648773193359, + "step": 47 + }, + { + "epoch": 0.5062623599208965, + "grad_norm": 3.124903678894043, + "learning_rate": 1.944806046466878e-05, + "loss": 0.9546631574630737, + "step": 48 + }, + { + "epoch": 0.5168094924192486, + "grad_norm": 3.41595458984375, + "learning_rate": 1.9381913359224844e-05, + "loss": 0.7677831649780273, + "step": 49 + }, + { + "epoch": 0.5273566249176005, + "grad_norm": 2.969999313354492, + "learning_rate": 1.9312149347588035e-05, + "loss": 0.8908669352531433, + "step": 50 + }, + { + "epoch": 0.5273566249176005, + "eval_loss": 0.7529963850975037, + "eval_runtime": 28.3177, + "eval_samples_per_second": 5.05, + "eval_steps_per_second": 1.271, + "step": 50 + }, + { + "epoch": 0.5379037574159525, + "grad_norm": 2.576383113861084, + "learning_rate": 1.9238795325112867e-05, + "loss": 0.7306627631187439, + "step": 51 + }, + { + "epoch": 0.5484508899143046, + "grad_norm": 2.5779168605804443, + "learning_rate": 1.916187957117136e-05, + "loss": 0.6502229571342468, + "step": 52 + }, + { + "epoch": 0.5589980224126566, + "grad_norm": 3.7034096717834473, + "learning_rate": 1.9081431738250815e-05, + "loss": 1.0519064664840698, + "step": 53 + }, + { + "epoch": 0.5695451549110085, + "grad_norm": 3.7297651767730713, + "learning_rate": 1.8997482840522218e-05, + "loss": 0.9438788890838623, + "step": 54 + }, + { + "epoch": 0.5800922874093606, + "grad_norm": 2.983699083328247, + "learning_rate": 1.891006524188368e-05, + "loss": 0.810992419719696, + "step": 55 + }, + { + "epoch": 0.5906394199077126, + "grad_norm": 2.869706869125366, + "learning_rate": 1.881921264348355e-05, + "loss": 0.681011438369751, + "step": 56 + }, + { + "epoch": 0.6011865524060646, + "grad_norm": 2.8603360652923584, + "learning_rate": 1.8724960070727974e-05, + "loss": 0.8332223892211914, + "step": 57 + }, + { + "epoch": 0.6117336849044166, + "grad_norm": 2.766416072845459, + "learning_rate": 1.862734385977792e-05, + "loss": 0.8047686815261841, + "step": 58 + }, + { + "epoch": 0.6222808174027686, + "grad_norm": 2.664792776107788, + "learning_rate": 1.8526401643540924e-05, + "loss": 0.6811239719390869, + "step": 59 + }, + { + "epoch": 0.6328279499011207, + "grad_norm": 2.787421226501465, + "learning_rate": 1.8422172337162865e-05, + "loss": 0.9071734547615051, + "step": 60 + }, + { + "epoch": 0.6433750823994726, + "grad_norm": 2.5916903018951416, + "learning_rate": 1.8314696123025456e-05, + "loss": 0.778200089931488, + "step": 61 + }, + { + "epoch": 0.6539222148978246, + "grad_norm": 2.5858092308044434, + "learning_rate": 1.8204014435255136e-05, + "loss": 0.7620019912719727, + "step": 62 + }, + { + "epoch": 0.6644693473961767, + "grad_norm": 3.226935386657715, + "learning_rate": 1.8090169943749477e-05, + "loss": 1.0030763149261475, + "step": 63 + }, + { + "epoch": 0.6750164798945286, + "grad_norm": 2.46641206741333, + "learning_rate": 1.797320653772707e-05, + "loss": 0.6492084860801697, + "step": 64 + }, + { + "epoch": 0.6855636123928807, + "grad_norm": 2.243069648742676, + "learning_rate": 1.785316930880745e-05, + "loss": 0.6458125114440918, + "step": 65 + }, + { + "epoch": 0.6961107448912327, + "grad_norm": 2.7569828033447266, + "learning_rate": 1.773010453362737e-05, + "loss": 0.8067665100097656, + "step": 66 + }, + { + "epoch": 0.7066578773895847, + "grad_norm": 3.01151967048645, + "learning_rate": 1.7604059656000313e-05, + "loss": 0.8096626996994019, + "step": 67 + }, + { + "epoch": 0.7172050098879367, + "grad_norm": 2.5362462997436523, + "learning_rate": 1.747508326862597e-05, + "loss": 0.5855382084846497, + "step": 68 + }, + { + "epoch": 0.7277521423862887, + "grad_norm": 3.2442595958709717, + "learning_rate": 1.7343225094356857e-05, + "loss": 1.0010881423950195, + "step": 69 + }, + { + "epoch": 0.7382992748846408, + "grad_norm": 2.9444711208343506, + "learning_rate": 1.720853596702919e-05, + "loss": 0.9237573146820068, + "step": 70 + }, + { + "epoch": 0.7488464073829928, + "grad_norm": 2.2847354412078857, + "learning_rate": 1.7071067811865477e-05, + "loss": 0.6892184615135193, + "step": 71 + }, + { + "epoch": 0.7593935398813447, + "grad_norm": 2.72588849067688, + "learning_rate": 1.6930873625456362e-05, + "loss": 0.7187391519546509, + "step": 72 + }, + { + "epoch": 0.7699406723796968, + "grad_norm": 2.462245225906372, + "learning_rate": 1.678800745532942e-05, + "loss": 0.6943771839141846, + "step": 73 + }, + { + "epoch": 0.7804878048780488, + "grad_norm": 2.880293846130371, + "learning_rate": 1.664252437911282e-05, + "loss": 0.7184453010559082, + "step": 74 + }, + { + "epoch": 0.7910349373764007, + "grad_norm": 3.225119113922119, + "learning_rate": 1.6494480483301836e-05, + "loss": 0.9926764369010925, + "step": 75 + }, + { + "epoch": 0.8015820698747528, + "grad_norm": 2.788250207901001, + "learning_rate": 1.6343932841636455e-05, + "loss": 0.8473926186561584, + "step": 76 + }, + { + "epoch": 0.8121292023731048, + "grad_norm": 2.1925415992736816, + "learning_rate": 1.6190939493098344e-05, + "loss": 0.5790418982505798, + "step": 77 + }, + { + "epoch": 0.8226763348714569, + "grad_norm": 2.4412641525268555, + "learning_rate": 1.6035559419535714e-05, + "loss": 0.677460789680481, + "step": 78 + }, + { + "epoch": 0.8332234673698088, + "grad_norm": 2.478698968887329, + "learning_rate": 1.5877852522924733e-05, + "loss": 0.8228996992111206, + "step": 79 + }, + { + "epoch": 0.8437705998681608, + "grad_norm": 2.5369465351104736, + "learning_rate": 1.5717879602276123e-05, + "loss": 0.7633792757987976, + "step": 80 + }, + { + "epoch": 0.8543177323665129, + "grad_norm": 2.9730212688446045, + "learning_rate": 1.5555702330196024e-05, + "loss": 1.0669782161712646, + "step": 81 + }, + { + "epoch": 0.8648648648648649, + "grad_norm": 2.1946942806243896, + "learning_rate": 1.5391383229110005e-05, + "loss": 0.6726582646369934, + "step": 82 + }, + { + "epoch": 0.8754119973632168, + "grad_norm": 2.4897115230560303, + "learning_rate": 1.5224985647159489e-05, + "loss": 0.7702843546867371, + "step": 83 + }, + { + "epoch": 0.8859591298615689, + "grad_norm": 2.055375337600708, + "learning_rate": 1.5056573733779848e-05, + "loss": 0.49126309156417847, + "step": 84 + }, + { + "epoch": 0.8965062623599209, + "grad_norm": 2.346480131149292, + "learning_rate": 1.4886212414969551e-05, + "loss": 0.7343087196350098, + "step": 85 + }, + { + "epoch": 0.9070533948582729, + "grad_norm": 2.2193572521209717, + "learning_rate": 1.4713967368259981e-05, + "loss": 0.5628997683525085, + "step": 86 + }, + { + "epoch": 0.9176005273566249, + "grad_norm": 2.818474292755127, + "learning_rate": 1.4539904997395468e-05, + "loss": 0.9049317240715027, + "step": 87 + }, + { + "epoch": 0.9281476598549769, + "grad_norm": 2.0409064292907715, + "learning_rate": 1.436409240673342e-05, + "loss": 0.609929621219635, + "step": 88 + }, + { + "epoch": 0.938694792353329, + "grad_norm": 2.5648860931396484, + "learning_rate": 1.4186597375374283e-05, + "loss": 0.7217040657997131, + "step": 89 + }, + { + "epoch": 0.9492419248516809, + "grad_norm": 2.435124158859253, + "learning_rate": 1.4007488331031409e-05, + "loss": 0.6683085560798645, + "step": 90 + }, + { + "epoch": 0.959789057350033, + "grad_norm": 2.642301082611084, + "learning_rate": 1.3826834323650899e-05, + "loss": 0.7930483222007751, + "step": 91 + }, + { + "epoch": 0.970336189848385, + "grad_norm": 2.8316915035247803, + "learning_rate": 1.3644704998791501e-05, + "loss": 0.8307502269744873, + "step": 92 + }, + { + "epoch": 0.980883322346737, + "grad_norm": 2.585615873336792, + "learning_rate": 1.346117057077493e-05, + "loss": 0.809612512588501, + "step": 93 + }, + { + "epoch": 0.991430454845089, + "grad_norm": 2.4871203899383545, + "learning_rate": 1.3276301795616937e-05, + "loss": 0.7022823095321655, + "step": 94 + }, + { + "epoch": 1.0, + "grad_norm": 2.2630867958068848, + "learning_rate": 1.3090169943749475e-05, + "loss": 0.540604293346405, + "step": 95 + }, + { + "epoch": 1.010547132498352, + "grad_norm": 1.8352792263031006, + "learning_rate": 1.2902846772544625e-05, + "loss": 0.47405001521110535, + "step": 96 + }, + { + "epoch": 1.021094264996704, + "grad_norm": 2.3510472774505615, + "learning_rate": 1.2714404498650743e-05, + "loss": 0.7485337257385254, + "step": 97 + }, + { + "epoch": 1.031641397495056, + "grad_norm": 2.6154630184173584, + "learning_rate": 1.252491577015158e-05, + "loss": 0.8424244523048401, + "step": 98 + }, + { + "epoch": 1.042188529993408, + "grad_norm": 2.235624074935913, + "learning_rate": 1.2334453638559057e-05, + "loss": 0.5035255551338196, + "step": 99 + }, + { + "epoch": 1.05273566249176, + "grad_norm": 2.5067241191864014, + "learning_rate": 1.2143091530650508e-05, + "loss": 0.839946985244751, + "step": 100 + }, + { + "epoch": 1.05273566249176, + "eval_loss": 0.69390469789505, + "eval_runtime": 28.1554, + "eval_samples_per_second": 5.079, + "eval_steps_per_second": 1.279, + "step": 100 + }, + { + "epoch": 1.063282794990112, + "grad_norm": 2.4152822494506836, + "learning_rate": 1.1950903220161286e-05, + "loss": 0.6019119024276733, + "step": 101 + }, + { + "epoch": 1.0738299274884642, + "grad_norm": 3.1178929805755615, + "learning_rate": 1.1757962799343548e-05, + "loss": 0.8134055733680725, + "step": 102 + }, + { + "epoch": 1.084377059986816, + "grad_norm": 2.630704402923584, + "learning_rate": 1.156434465040231e-05, + "loss": 0.7413411736488342, + "step": 103 + }, + { + "epoch": 1.094924192485168, + "grad_norm": 2.8832743167877197, + "learning_rate": 1.1370123416819683e-05, + "loss": 0.8591673374176025, + "step": 104 + }, + { + "epoch": 1.1054713249835202, + "grad_norm": 2.6520588397979736, + "learning_rate": 1.1175373974578378e-05, + "loss": 0.7692611813545227, + "step": 105 + }, + { + "epoch": 1.1160184574818721, + "grad_norm": 2.3777475357055664, + "learning_rate": 1.098017140329561e-05, + "loss": 0.644071102142334, + "step": 106 + }, + { + "epoch": 1.126565589980224, + "grad_norm": 2.3438470363616943, + "learning_rate": 1.0784590957278452e-05, + "loss": 0.6566027998924255, + "step": 107 + }, + { + "epoch": 1.1371127224785762, + "grad_norm": 2.494967222213745, + "learning_rate": 1.058870803651189e-05, + "loss": 0.6349762678146362, + "step": 108 + }, + { + "epoch": 1.1476598549769281, + "grad_norm": 2.557257652282715, + "learning_rate": 1.0392598157590687e-05, + "loss": 0.7407976388931274, + "step": 109 + }, + { + "epoch": 1.15820698747528, + "grad_norm": 2.54610013961792, + "learning_rate": 1.0196336924606282e-05, + "loss": 0.6565055251121521, + "step": 110 + }, + { + "epoch": 1.1687541199736322, + "grad_norm": 2.7403500080108643, + "learning_rate": 1e-05, + "loss": 0.9395892024040222, + "step": 111 + }, + { + "epoch": 1.1793012524719841, + "grad_norm": 2.6010167598724365, + "learning_rate": 9.80366307539372e-06, + "loss": 0.7564979791641235, + "step": 112 + }, + { + "epoch": 1.189848384970336, + "grad_norm": 2.5348997116088867, + "learning_rate": 9.607401842409318e-06, + "loss": 0.671207070350647, + "step": 113 + }, + { + "epoch": 1.2003955174686882, + "grad_norm": 2.2326834201812744, + "learning_rate": 9.41129196348811e-06, + "loss": 0.4958484470844269, + "step": 114 + }, + { + "epoch": 1.2109426499670402, + "grad_norm": 2.7414183616638184, + "learning_rate": 9.215409042721553e-06, + "loss": 0.7510374784469604, + "step": 115 + }, + { + "epoch": 1.2214897824653923, + "grad_norm": 2.51889705657959, + "learning_rate": 9.019828596704394e-06, + "loss": 0.6310056447982788, + "step": 116 + }, + { + "epoch": 1.2320369149637442, + "grad_norm": 2.1737775802612305, + "learning_rate": 8.824626025421625e-06, + "loss": 0.5721197128295898, + "step": 117 + }, + { + "epoch": 1.2425840474620962, + "grad_norm": 2.6601269245147705, + "learning_rate": 8.629876583180322e-06, + "loss": 0.6723504662513733, + "step": 118 + }, + { + "epoch": 1.2531311799604483, + "grad_norm": 3.153501033782959, + "learning_rate": 8.43565534959769e-06, + "loss": 0.8398867845535278, + "step": 119 + }, + { + "epoch": 1.2636783124588002, + "grad_norm": 2.913832664489746, + "learning_rate": 8.242037200656455e-06, + "loss": 0.7067492008209229, + "step": 120 + }, + { + "epoch": 1.2742254449571524, + "grad_norm": 3.4029502868652344, + "learning_rate": 8.04909677983872e-06, + "loss": 0.9664009809494019, + "step": 121 + }, + { + "epoch": 1.2847725774555043, + "grad_norm": 2.364175319671631, + "learning_rate": 7.856908469349495e-06, + "loss": 0.5740135312080383, + "step": 122 + }, + { + "epoch": 1.2953197099538563, + "grad_norm": 2.6324050426483154, + "learning_rate": 7.66554636144095e-06, + "loss": 0.6533321738243103, + "step": 123 + }, + { + "epoch": 1.3058668424522084, + "grad_norm": 2.7875795364379883, + "learning_rate": 7.4750842298484205e-06, + "loss": 0.7847114205360413, + "step": 124 + }, + { + "epoch": 1.3164139749505603, + "grad_norm": 2.482607126235962, + "learning_rate": 7.285595501349259e-06, + "loss": 0.6538381576538086, + "step": 125 + }, + { + "epoch": 1.3269611074489123, + "grad_norm": 2.798854351043701, + "learning_rate": 7.097153227455379e-06, + "loss": 0.7187844514846802, + "step": 126 + }, + { + "epoch": 1.3375082399472644, + "grad_norm": 3.029470920562744, + "learning_rate": 6.909830056250527e-06, + "loss": 0.859753429889679, + "step": 127 + }, + { + "epoch": 1.3480553724456164, + "grad_norm": 2.3846898078918457, + "learning_rate": 6.723698204383067e-06, + "loss": 0.6410857439041138, + "step": 128 + }, + { + "epoch": 1.3586025049439683, + "grad_norm": 2.828608751296997, + "learning_rate": 6.538829429225068e-06, + "loss": 0.7521955966949463, + "step": 129 + }, + { + "epoch": 1.3691496374423204, + "grad_norm": 2.1056339740753174, + "learning_rate": 6.355295001208504e-06, + "loss": 0.49943816661834717, + "step": 130 + }, + { + "epoch": 1.3796967699406724, + "grad_norm": 2.9909141063690186, + "learning_rate": 6.173165676349103e-06, + "loss": 0.6608769297599792, + "step": 131 + }, + { + "epoch": 1.3902439024390243, + "grad_norm": 2.3114278316497803, + "learning_rate": 5.9925116689685925e-06, + "loss": 0.5453819632530212, + "step": 132 + }, + { + "epoch": 1.4007910349373764, + "grad_norm": 2.200542688369751, + "learning_rate": 5.813402624625722e-06, + "loss": 0.4996246099472046, + "step": 133 + }, + { + "epoch": 1.4113381674357284, + "grad_norm": 2.2516205310821533, + "learning_rate": 5.635907593266578e-06, + "loss": 0.63039630651474, + "step": 134 + }, + { + "epoch": 1.4218852999340803, + "grad_norm": 2.5531094074249268, + "learning_rate": 5.460095002604533e-06, + "loss": 0.65385502576828, + "step": 135 + }, + { + "epoch": 1.4324324324324325, + "grad_norm": 2.369076728820801, + "learning_rate": 5.286032631740023e-06, + "loss": 0.5936092138290405, + "step": 136 + }, + { + "epoch": 1.4429795649307844, + "grad_norm": 2.984978199005127, + "learning_rate": 5.1137875850304545e-06, + "loss": 0.7236067652702332, + "step": 137 + }, + { + "epoch": 1.4535266974291363, + "grad_norm": 2.5663387775421143, + "learning_rate": 4.943426266220156e-06, + "loss": 0.6463375091552734, + "step": 138 + }, + { + "epoch": 1.4640738299274885, + "grad_norm": 2.425461769104004, + "learning_rate": 4.775014352840512e-06, + "loss": 0.6277223825454712, + "step": 139 + }, + { + "epoch": 1.4746209624258404, + "grad_norm": 2.8183038234710693, + "learning_rate": 4.608616770889998e-06, + "loss": 0.8176087141036987, + "step": 140 + }, + { + "epoch": 1.4851680949241926, + "grad_norm": 2.203562021255493, + "learning_rate": 4.444297669803981e-06, + "loss": 0.4891142249107361, + "step": 141 + }, + { + "epoch": 1.4957152274225445, + "grad_norm": 2.412172794342041, + "learning_rate": 4.282120397723879e-06, + "loss": 0.735496997833252, + "step": 142 + }, + { + "epoch": 1.5062623599208966, + "grad_norm": 2.850745916366577, + "learning_rate": 4.12214747707527e-06, + "loss": 0.736525297164917, + "step": 143 + }, + { + "epoch": 1.5168094924192486, + "grad_norm": 2.6273257732391357, + "learning_rate": 3.964440580464286e-06, + "loss": 0.629302978515625, + "step": 144 + }, + { + "epoch": 1.5273566249176005, + "grad_norm": 2.045539379119873, + "learning_rate": 3.8090605069016596e-06, + "loss": 0.5002003312110901, + "step": 145 + }, + { + "epoch": 1.5379037574159526, + "grad_norm": 2.5290746688842773, + "learning_rate": 3.6560671583635467e-06, + "loss": 0.604846715927124, + "step": 146 + }, + { + "epoch": 1.5484508899143046, + "grad_norm": 2.6563849449157715, + "learning_rate": 3.505519516698165e-06, + "loss": 0.7351030111312866, + "step": 147 + }, + { + "epoch": 1.5589980224126565, + "grad_norm": 2.430051326751709, + "learning_rate": 3.3574756208871862e-06, + "loss": 0.49856996536254883, + "step": 148 + }, + { + "epoch": 1.5695451549110087, + "grad_norm": 2.4349019527435303, + "learning_rate": 3.2119925446705824e-06, + "loss": 0.6530035138130188, + "step": 149 + }, + { + "epoch": 1.5800922874093606, + "grad_norm": 2.302081823348999, + "learning_rate": 3.069126374543643e-06, + "loss": 0.5958735346794128, + "step": 150 + }, + { + "epoch": 1.5800922874093606, + "eval_loss": 0.6550542712211609, + "eval_runtime": 27.788, + "eval_samples_per_second": 5.146, + "eval_steps_per_second": 1.296, + "step": 150 + }, + { + "epoch": 1.5906394199077125, + "grad_norm": 2.50254487991333, + "learning_rate": 2.9289321881345257e-06, + "loss": 0.6623690128326416, + "step": 151 + }, + { + "epoch": 1.6011865524060647, + "grad_norm": 2.416368007659912, + "learning_rate": 2.791464032970812e-06, + "loss": 0.6781509518623352, + "step": 152 + }, + { + "epoch": 1.6117336849044166, + "grad_norm": 2.349985361099243, + "learning_rate": 2.656774905643147e-06, + "loss": 0.5873178243637085, + "step": 153 + }, + { + "epoch": 1.6222808174027685, + "grad_norm": 2.329050064086914, + "learning_rate": 2.5249167313740307e-06, + "loss": 0.5800527930259705, + "step": 154 + }, + { + "epoch": 1.6328279499011207, + "grad_norm": 3.402740001678467, + "learning_rate": 2.395940343999691e-06, + "loss": 0.9891058802604675, + "step": 155 + }, + { + "epoch": 1.6433750823994726, + "grad_norm": 2.339655876159668, + "learning_rate": 2.26989546637263e-06, + "loss": 0.5989134907722473, + "step": 156 + }, + { + "epoch": 1.6539222148978245, + "grad_norm": 2.4846158027648926, + "learning_rate": 2.146830691192553e-06, + "loss": 0.6990019083023071, + "step": 157 + }, + { + "epoch": 1.6644693473961767, + "grad_norm": 2.869048833847046, + "learning_rate": 2.02679346227293e-06, + "loss": 0.7427476048469543, + "step": 158 + }, + { + "epoch": 1.6750164798945286, + "grad_norm": 2.9657325744628906, + "learning_rate": 1.9098300562505266e-06, + "loss": 0.7665688991546631, + "step": 159 + }, + { + "epoch": 1.6855636123928806, + "grad_norm": 2.206305980682373, + "learning_rate": 1.7959855647448642e-06, + "loss": 0.4983620345592499, + "step": 160 + }, + { + "epoch": 1.6961107448912327, + "grad_norm": 3.029860734939575, + "learning_rate": 1.6853038769745466e-06, + "loss": 0.7966647148132324, + "step": 161 + }, + { + "epoch": 1.7066578773895849, + "grad_norm": 2.863281488418579, + "learning_rate": 1.577827662837136e-06, + "loss": 0.7898584008216858, + "step": 162 + }, + { + "epoch": 1.7172050098879366, + "grad_norm": 2.178529977798462, + "learning_rate": 1.4735983564590784e-06, + "loss": 0.43844425678253174, + "step": 163 + }, + { + "epoch": 1.7277521423862887, + "grad_norm": 2.6796844005584717, + "learning_rate": 1.3726561402220818e-06, + "loss": 0.720207154750824, + "step": 164 + }, + { + "epoch": 1.7382992748846409, + "grad_norm": 2.2868733406066895, + "learning_rate": 1.2750399292720284e-06, + "loss": 0.5092083215713501, + "step": 165 + }, + { + "epoch": 1.7488464073829928, + "grad_norm": 2.885493278503418, + "learning_rate": 1.1807873565164507e-06, + "loss": 0.8737746477127075, + "step": 166 + }, + { + "epoch": 1.7593935398813447, + "grad_norm": 2.6771187782287598, + "learning_rate": 1.0899347581163222e-06, + "loss": 0.7444260716438293, + "step": 167 + }, + { + "epoch": 1.7699406723796969, + "grad_norm": 2.8727669715881348, + "learning_rate": 1.0025171594777872e-06, + "loss": 0.6898815035820007, + "step": 168 + }, + { + "epoch": 1.7804878048780488, + "grad_norm": 2.0689499378204346, + "learning_rate": 9.185682617491865e-07, + "loss": 0.5056309103965759, + "step": 169 + }, + { + "epoch": 1.7910349373764007, + "grad_norm": 2.400623083114624, + "learning_rate": 8.381204288286415e-07, + "loss": 0.6324359774589539, + "step": 170 + }, + { + "epoch": 1.801582069874753, + "grad_norm": 2.1962051391601562, + "learning_rate": 7.612046748871327e-07, + "loss": 0.4860890507698059, + "step": 171 + }, + { + "epoch": 1.8121292023731048, + "grad_norm": 2.8626863956451416, + "learning_rate": 6.878506524119644e-07, + "loss": 0.7613487243652344, + "step": 172 + }, + { + "epoch": 1.8226763348714567, + "grad_norm": 2.8272652626037598, + "learning_rate": 6.180866407751595e-07, + "loss": 0.8311713933944702, + "step": 173 + }, + { + "epoch": 1.833223467369809, + "grad_norm": 2.269026517868042, + "learning_rate": 5.519395353312195e-07, + "loss": 0.5173636078834534, + "step": 174 + }, + { + "epoch": 1.8437705998681608, + "grad_norm": 2.7205731868743896, + "learning_rate": 4.894348370484648e-07, + "loss": 0.6850346922874451, + "step": 175 + }, + { + "epoch": 1.8543177323665128, + "grad_norm": 2.6293232440948486, + "learning_rate": 4.305966426779118e-07, + "loss": 0.5403839349746704, + "step": 176 + }, + { + "epoch": 1.864864864864865, + "grad_norm": 3.077864408493042, + "learning_rate": 3.7544763546352834e-07, + "loss": 0.8380445241928101, + "step": 177 + }, + { + "epoch": 1.8754119973632168, + "grad_norm": 2.4063024520874023, + "learning_rate": 3.2400907639740243e-07, + "loss": 0.5892568826675415, + "step": 178 + }, + { + "epoch": 1.8859591298615688, + "grad_norm": 2.6916375160217285, + "learning_rate": 2.7630079602323447e-07, + "loss": 0.6844539642333984, + "step": 179 + }, + { + "epoch": 1.896506262359921, + "grad_norm": 2.5491273403167725, + "learning_rate": 2.3234118679127615e-07, + "loss": 0.653813898563385, + "step": 180 + }, + { + "epoch": 1.9070533948582729, + "grad_norm": 2.637279987335205, + "learning_rate": 1.921471959676957e-07, + "loss": 0.6304491758346558, + "step": 181 + }, + { + "epoch": 1.9176005273566248, + "grad_norm": 2.418591260910034, + "learning_rate": 1.5573431910108404e-07, + "loss": 0.6059620380401611, + "step": 182 + }, + { + "epoch": 1.928147659854977, + "grad_norm": 3.2978029251098633, + "learning_rate": 1.231165940486234e-07, + "loss": 0.9622435569763184, + "step": 183 + }, + { + "epoch": 1.938694792353329, + "grad_norm": 2.8531460762023926, + "learning_rate": 9.43065955642275e-08, + "loss": 0.8542320728302002, + "step": 184 + }, + { + "epoch": 1.9492419248516808, + "grad_norm": 3.4283599853515625, + "learning_rate": 6.931543045073708e-08, + "loss": 0.86351078748703, + "step": 185 + }, + { + "epoch": 1.959789057350033, + "grad_norm": 2.7210655212402344, + "learning_rate": 4.815273327803183e-08, + "loss": 0.7165011763572693, + "step": 186 + }, + { + "epoch": 1.970336189848385, + "grad_norm": 2.270888328552246, + "learning_rate": 3.082666266872036e-08, + "loss": 0.5722820162773132, + "step": 187 + }, + { + "epoch": 1.980883322346737, + "grad_norm": 2.5142128467559814, + "learning_rate": 1.7343898152841765e-08, + "loss": 0.6186709403991699, + "step": 188 + }, + { + "epoch": 1.991430454845089, + "grad_norm": 2.536613941192627, + "learning_rate": 7.70963759277099e-09, + "loss": 0.6009924411773682, + "step": 189 + }, + { + "epoch": 2.0, + "grad_norm": 2.4339284896850586, + "learning_rate": 1.9275951793518154e-09, + "loss": 0.5079313516616821, + "step": 190 + }, + { + "epoch": 2.0, + "eval_loss": 0.6482135057449341, + "eval_runtime": 27.7492, + "eval_samples_per_second": 5.153, + "eval_steps_per_second": 1.297, + "step": 190 + } + ], + "logging_steps": 1, + "max_steps": 190, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 5.454107883491328e+16, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/lora_adapter/checkpoint-190/training_args.bin b/lora_adapter/checkpoint-190/training_args.bin new file mode 100644 index 0000000..9b1b43f --- /dev/null +++ b/lora_adapter/checkpoint-190/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a35a78b99e3a3a7f629b461e552407e5f338f437d3bf7df51fb87ffb0945bec7 +size 5713 diff --git a/lora_adapter/checkpoint-250/README.md b/lora_adapter/checkpoint-250/README.md new file mode 100644 index 0000000..a4c765d --- /dev/null +++ b/lora_adapter/checkpoint-250/README.md @@ -0,0 +1,210 @@ +--- +base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +- lora +- sft +- transformers +- trl +- unsloth +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/lora_adapter/checkpoint-250/adapter_config.json b/lora_adapter/checkpoint-250/adapter_config.json new file mode 100644 index 0000000..6e70f33 --- /dev/null +++ b/lora_adapter/checkpoint-250/adapter_config.json @@ -0,0 +1,52 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": { + "base_model_class": "Qwen2ForCausalLM", + "parent_library": "transformers.models.qwen2.modeling_qwen2", + "unsloth_fixed": true + }, + "base_model_name_or_path": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "up_proj", + "gate_proj", + "q_proj", + "v_proj", + "down_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": true +} \ No newline at end of file diff --git a/lora_adapter/checkpoint-250/adapter_model.safetensors b/lora_adapter/checkpoint-250/adapter_model.safetensors new file mode 100644 index 0000000..a470896 --- /dev/null +++ b/lora_adapter/checkpoint-250/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54790ab46c04ffca134c4f425d26fa579a9e7fb7c24ebc7652ae2aaf1a136121 +size 295488936 diff --git a/lora_adapter/checkpoint-250/chat_template.jinja b/lora_adapter/checkpoint-250/chat_template.jinja new file mode 100644 index 0000000..642e597 --- /dev/null +++ b/lora_adapter/checkpoint-250/chat_template.jinja @@ -0,0 +1,53 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/lora_adapter/checkpoint-250/optimizer.pt b/lora_adapter/checkpoint-250/optimizer.pt new file mode 100644 index 0000000..a3767b1 --- /dev/null +++ b/lora_adapter/checkpoint-250/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c48844d856e7b232444ea0a833bee3a7b3502c89778123b39ec0a2528603c433 +size 150491333 diff --git a/lora_adapter/checkpoint-250/rng_state.pth b/lora_adapter/checkpoint-250/rng_state.pth new file mode 100644 index 0000000..1fb544a --- /dev/null +++ b/lora_adapter/checkpoint-250/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdc37bbd2e979f041dfbbb004a5c74bab6cdda159cb18116df728588515a9ef6 +size 14645 diff --git a/lora_adapter/checkpoint-250/scheduler.pt b/lora_adapter/checkpoint-250/scheduler.pt new file mode 100644 index 0000000..bec3ec3 --- /dev/null +++ b/lora_adapter/checkpoint-250/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6db5e416dcafe4ae2b6b2ae3b96e6546a75cfdde7db951b8d58b87f9c2507ee5 +size 1465 diff --git a/lora_adapter/checkpoint-250/tokenizer.json b/lora_adapter/checkpoint-250/tokenizer.json new file mode 100644 index 0000000..5340d81 --- /dev/null +++ b/lora_adapter/checkpoint-250/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd5948af71b4f56cf697f7580814c7ce8b80595ef985544efcacf716126a2e31 +size 11422356 diff --git a/lora_adapter/checkpoint-250/tokenizer_config.json b/lora_adapter/checkpoint-250/tokenizer_config.json new file mode 100644 index 0000000..d3dac53 --- /dev/null +++ b/lora_adapter/checkpoint-250/tokenizer_config.json @@ -0,0 +1,201 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 32768, + "pad_token": "<|PAD_TOKEN|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151657": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151658": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151665": { + "content": "<|PAD_TOKEN|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + } +} diff --git a/lora_adapter/checkpoint-250/trainer_state.json b/lora_adapter/checkpoint-250/trainer_state.json new file mode 100644 index 0000000..f71131e --- /dev/null +++ b/lora_adapter/checkpoint-250/trainer_state.json @@ -0,0 +1,1824 @@ +{ + "best_global_step": 250, + "best_metric": 0.7293056845664978, + "best_model_checkpoint": "./qwen_unified_finetuned/checkpoint-250", + "epoch": 2.817601135557133, + "eval_steps": 50, + "global_step": 250, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0113555713271824, + "grad_norm": 3.916877031326294, + "learning_rate": 0.0, + "loss": 1.048294186592102, + "step": 1 + }, + { + "epoch": 0.0227111426543648, + "grad_norm": 4.441281318664551, + "learning_rate": 6.666666666666667e-07, + "loss": 0.8231856822967529, + "step": 2 + }, + { + "epoch": 0.034066713981547196, + "grad_norm": 4.950929641723633, + "learning_rate": 1.3333333333333334e-06, + "loss": 1.31972074508667, + "step": 3 + }, + { + "epoch": 0.0454222853087296, + "grad_norm": 4.647597789764404, + "learning_rate": 2.0000000000000003e-06, + "loss": 1.1833959817886353, + "step": 4 + }, + { + "epoch": 0.05677785663591199, + "grad_norm": 4.787125110626221, + "learning_rate": 2.666666666666667e-06, + "loss": 1.5079869031906128, + "step": 5 + }, + { + "epoch": 0.06813342796309439, + "grad_norm": 4.424062252044678, + "learning_rate": 3.3333333333333333e-06, + "loss": 1.0498075485229492, + "step": 6 + }, + { + "epoch": 0.0794889992902768, + "grad_norm": 4.251436710357666, + "learning_rate": 4.000000000000001e-06, + "loss": 1.0172538757324219, + "step": 7 + }, + { + "epoch": 0.0908445706174592, + "grad_norm": 3.5152761936187744, + "learning_rate": 4.666666666666667e-06, + "loss": 1.1070594787597656, + "step": 8 + }, + { + "epoch": 0.1022001419446416, + "grad_norm": 3.2286136150360107, + "learning_rate": 5.333333333333334e-06, + "loss": 0.8639000058174133, + "step": 9 + }, + { + "epoch": 0.11355571327182398, + "grad_norm": 2.9262876510620117, + "learning_rate": 6e-06, + "loss": 0.927219033241272, + "step": 10 + }, + { + "epoch": 0.12491128459900638, + "grad_norm": 3.439363956451416, + "learning_rate": 6.666666666666667e-06, + "loss": 1.2999275922775269, + "step": 11 + }, + { + "epoch": 0.13626685592618878, + "grad_norm": 2.8811707496643066, + "learning_rate": 7.333333333333333e-06, + "loss": 1.0791280269622803, + "step": 12 + }, + { + "epoch": 0.14762242725337119, + "grad_norm": 2.5855603218078613, + "learning_rate": 8.000000000000001e-06, + "loss": 1.0025461912155151, + "step": 13 + }, + { + "epoch": 0.1589779985805536, + "grad_norm": 2.93908953666687, + "learning_rate": 8.666666666666668e-06, + "loss": 1.2033611536026, + "step": 14 + }, + { + "epoch": 0.170333569907736, + "grad_norm": 2.1815803050994873, + "learning_rate": 9.333333333333334e-06, + "loss": 0.865071177482605, + "step": 15 + }, + { + "epoch": 0.1816891412349184, + "grad_norm": 2.5811753273010254, + "learning_rate": 1e-05, + "loss": 0.9821955561637878, + "step": 16 + }, + { + "epoch": 0.1930447125621008, + "grad_norm": 2.2700631618499756, + "learning_rate": 1.0666666666666667e-05, + "loss": 0.943819522857666, + "step": 17 + }, + { + "epoch": 0.2044002838892832, + "grad_norm": 2.234851360321045, + "learning_rate": 1.1333333333333334e-05, + "loss": 0.8833191990852356, + "step": 18 + }, + { + "epoch": 0.2157558552164656, + "grad_norm": 2.1426126956939697, + "learning_rate": 1.2e-05, + "loss": 0.8689224720001221, + "step": 19 + }, + { + "epoch": 0.22711142654364797, + "grad_norm": 2.0993337631225586, + "learning_rate": 1.2666666666666667e-05, + "loss": 0.8528774380683899, + "step": 20 + }, + { + "epoch": 0.23846699787083037, + "grad_norm": 1.9766696691513062, + "learning_rate": 1.3333333333333333e-05, + "loss": 0.889632523059845, + "step": 21 + }, + { + "epoch": 0.24982256919801277, + "grad_norm": 2.3761353492736816, + "learning_rate": 1.4e-05, + "loss": 0.9641993045806885, + "step": 22 + }, + { + "epoch": 0.26117814052519517, + "grad_norm": 2.226921558380127, + "learning_rate": 1.4666666666666666e-05, + "loss": 1.0349667072296143, + "step": 23 + }, + { + "epoch": 0.27253371185237757, + "grad_norm": 1.9597960710525513, + "learning_rate": 1.5333333333333334e-05, + "loss": 0.8142374157905579, + "step": 24 + }, + { + "epoch": 0.28388928317955997, + "grad_norm": 1.769181728363037, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.6785438656806946, + "step": 25 + }, + { + "epoch": 0.29524485450674237, + "grad_norm": 2.046727418899536, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.9442430734634399, + "step": 26 + }, + { + "epoch": 0.3066004258339248, + "grad_norm": 2.2059895992279053, + "learning_rate": 1.7333333333333336e-05, + "loss": 1.0432536602020264, + "step": 27 + }, + { + "epoch": 0.3179559971611072, + "grad_norm": 2.3628499507904053, + "learning_rate": 1.8e-05, + "loss": 1.0692076683044434, + "step": 28 + }, + { + "epoch": 0.3293115684882896, + "grad_norm": 2.2492411136627197, + "learning_rate": 1.866666666666667e-05, + "loss": 1.1659228801727295, + "step": 29 + }, + { + "epoch": 0.340667139815472, + "grad_norm": 2.217177152633667, + "learning_rate": 1.9333333333333333e-05, + "loss": 0.706802487373352, + "step": 30 + }, + { + "epoch": 0.3520227111426544, + "grad_norm": 1.6823749542236328, + "learning_rate": 2e-05, + "loss": 0.7545626163482666, + "step": 31 + }, + { + "epoch": 0.3633782824698368, + "grad_norm": 2.0650510787963867, + "learning_rate": 1.9999121449564347e-05, + "loss": 0.8590264320373535, + "step": 32 + }, + { + "epoch": 0.3747338537970192, + "grad_norm": 1.6296216249465942, + "learning_rate": 1.9996485952627554e-05, + "loss": 0.7331938743591309, + "step": 33 + }, + { + "epoch": 0.3860894251242016, + "grad_norm": 1.7522505521774292, + "learning_rate": 1.999209397227302e-05, + "loss": 0.7665808796882629, + "step": 34 + }, + { + "epoch": 0.397444996451384, + "grad_norm": 2.035996198654175, + "learning_rate": 1.9985946280215996e-05, + "loss": 0.9990826845169067, + "step": 35 + }, + { + "epoch": 0.4088005677785664, + "grad_norm": 1.993740200996399, + "learning_rate": 1.997804395666799e-05, + "loss": 0.8873282074928284, + "step": 36 + }, + { + "epoch": 0.4201561391057488, + "grad_norm": 1.9349602460861206, + "learning_rate": 1.996838839014696e-05, + "loss": 0.8252230286598206, + "step": 37 + }, + { + "epoch": 0.4315117104329312, + "grad_norm": 2.1690878868103027, + "learning_rate": 1.9956981277233342e-05, + "loss": 1.0567874908447266, + "step": 38 + }, + { + "epoch": 0.44286728176011353, + "grad_norm": 2.005312442779541, + "learning_rate": 1.9943824622271934e-05, + "loss": 0.980779767036438, + "step": 39 + }, + { + "epoch": 0.45422285308729593, + "grad_norm": 2.0727038383483887, + "learning_rate": 1.9928920737019735e-05, + "loss": 0.8422978520393372, + "step": 40 + }, + { + "epoch": 0.46557842441447833, + "grad_norm": 2.4477312564849854, + "learning_rate": 1.9912272240239715e-05, + "loss": 1.239004373550415, + "step": 41 + }, + { + "epoch": 0.47693399574166073, + "grad_norm": 2.073049306869507, + "learning_rate": 1.9893882057240698e-05, + "loss": 1.0170605182647705, + "step": 42 + }, + { + "epoch": 0.48828956706884313, + "grad_norm": 1.9096711874008179, + "learning_rate": 1.9873753419363336e-05, + "loss": 0.7338559031486511, + "step": 43 + }, + { + "epoch": 0.49964513839602553, + "grad_norm": 1.514130711555481, + "learning_rate": 1.9851889863412347e-05, + "loss": 0.5721548795700073, + "step": 44 + }, + { + "epoch": 0.511000709723208, + "grad_norm": 1.8837509155273438, + "learning_rate": 1.9828295231035054e-05, + "loss": 0.7944353222846985, + "step": 45 + }, + { + "epoch": 0.5223562810503903, + "grad_norm": 1.6668717861175537, + "learning_rate": 1.9802973668046364e-05, + "loss": 0.703384518623352, + "step": 46 + }, + { + "epoch": 0.5337118523775728, + "grad_norm": 1.619913935661316, + "learning_rate": 1.9775929623700318e-05, + "loss": 0.7028679847717285, + "step": 47 + }, + { + "epoch": 0.5450674237047551, + "grad_norm": 1.797451138496399, + "learning_rate": 1.9747167849908305e-05, + "loss": 0.8451980948448181, + "step": 48 + }, + { + "epoch": 0.5564229950319376, + "grad_norm": 1.6591897010803223, + "learning_rate": 1.97166934004041e-05, + "loss": 0.6273907423019409, + "step": 49 + }, + { + "epoch": 0.5677785663591199, + "grad_norm": 1.6978833675384521, + "learning_rate": 1.968451162985589e-05, + "loss": 0.6259630918502808, + "step": 50 + }, + { + "epoch": 0.5677785663591199, + "eval_loss": 0.8047120571136475, + "eval_runtime": 27.6692, + "eval_samples_per_second": 5.168, + "eval_steps_per_second": 1.301, + "step": 50 + }, + { + "epoch": 0.5791341376863024, + "grad_norm": 2.0382018089294434, + "learning_rate": 1.9650628192925372e-05, + "loss": 0.8624173402786255, + "step": 51 + }, + { + "epoch": 0.5904897090134847, + "grad_norm": 2.2218334674835205, + "learning_rate": 1.9615049043274207e-05, + "loss": 1.0386168956756592, + "step": 52 + }, + { + "epoch": 0.6018452803406671, + "grad_norm": 1.7063782215118408, + "learning_rate": 1.957778043251788e-05, + "loss": 0.7194206118583679, + "step": 53 + }, + { + "epoch": 0.6132008516678495, + "grad_norm": 1.621178150177002, + "learning_rate": 1.953882890912723e-05, + "loss": 0.804326593875885, + "step": 54 + }, + { + "epoch": 0.6245564229950319, + "grad_norm": 1.5988848209381104, + "learning_rate": 1.949820131727783e-05, + "loss": 0.6942477822303772, + "step": 55 + }, + { + "epoch": 0.6359119943222143, + "grad_norm": 1.7638858556747437, + "learning_rate": 1.945590479564738e-05, + "loss": 0.688396692276001, + "step": 56 + }, + { + "epoch": 0.6472675656493967, + "grad_norm": 1.8592664003372192, + "learning_rate": 1.9411946776161388e-05, + "loss": 0.7517380118370056, + "step": 57 + }, + { + "epoch": 0.6586231369765791, + "grad_norm": 2.1580257415771484, + "learning_rate": 1.936633498268728e-05, + "loss": 1.1234959363937378, + "step": 58 + }, + { + "epoch": 0.6699787083037615, + "grad_norm": 2.0941476821899414, + "learning_rate": 1.931907742967727e-05, + "loss": 0.7969536185264587, + "step": 59 + }, + { + "epoch": 0.681334279630944, + "grad_norm": 1.8826228380203247, + "learning_rate": 1.9270182420760104e-05, + "loss": 0.7057633996009827, + "step": 60 + }, + { + "epoch": 0.6926898509581263, + "grad_norm": 1.8172886371612549, + "learning_rate": 1.921965854728207e-05, + "loss": 0.8432231545448303, + "step": 61 + }, + { + "epoch": 0.7040454222853088, + "grad_norm": 2.468703508377075, + "learning_rate": 1.916751468679737e-05, + "loss": 0.9716429710388184, + "step": 62 + }, + { + "epoch": 0.7154009936124911, + "grad_norm": 1.5727559328079224, + "learning_rate": 1.911376000150828e-05, + "loss": 0.6017226576805115, + "step": 63 + }, + { + "epoch": 0.7267565649396736, + "grad_norm": 1.9279130697250366, + "learning_rate": 1.9058403936655235e-05, + "loss": 0.945792019367218, + "step": 64 + }, + { + "epoch": 0.7381121362668559, + "grad_norm": 1.459025502204895, + "learning_rate": 1.9001456218857207e-05, + "loss": 0.601871907711029, + "step": 65 + }, + { + "epoch": 0.7494677075940384, + "grad_norm": 1.5340495109558105, + "learning_rate": 1.894292685440266e-05, + "loss": 0.6935733556747437, + "step": 66 + }, + { + "epoch": 0.7608232789212207, + "grad_norm": 1.6071991920471191, + "learning_rate": 1.888282612749132e-05, + "loss": 0.7782894968986511, + "step": 67 + }, + { + "epoch": 0.7721788502484032, + "grad_norm": 1.6310977935791016, + "learning_rate": 1.8821164598427148e-05, + "loss": 0.7309603095054626, + "step": 68 + }, + { + "epoch": 0.7835344215755855, + "grad_norm": 1.817807912826538, + "learning_rate": 1.8757953101762786e-05, + "loss": 0.790179967880249, + "step": 69 + }, + { + "epoch": 0.794889992902768, + "grad_norm": 1.3777976036071777, + "learning_rate": 1.869320274439583e-05, + "loss": 0.5821739435195923, + "step": 70 + }, + { + "epoch": 0.8062455642299503, + "grad_norm": 1.3368778228759766, + "learning_rate": 1.86269249036172e-05, + "loss": 0.5191158652305603, + "step": 71 + }, + { + "epoch": 0.8176011355571328, + "grad_norm": 1.7497543096542358, + "learning_rate": 1.8559131225112085e-05, + "loss": 0.7746997475624084, + "step": 72 + }, + { + "epoch": 0.8289567068843151, + "grad_norm": 1.6474581956863403, + "learning_rate": 1.8489833620913644e-05, + "loss": 0.5990749001502991, + "step": 73 + }, + { + "epoch": 0.8403122782114976, + "grad_norm": 1.425794005393982, + "learning_rate": 1.841904426730994e-05, + "loss": 0.6433415412902832, + "step": 74 + }, + { + "epoch": 0.8516678495386799, + "grad_norm": 1.8670772314071655, + "learning_rate": 1.8346775602704464e-05, + "loss": 0.840431809425354, + "step": 75 + }, + { + "epoch": 0.8630234208658624, + "grad_norm": 1.6109037399291992, + "learning_rate": 1.8273040325430575e-05, + "loss": 0.7096354961395264, + "step": 76 + }, + { + "epoch": 0.8743789921930447, + "grad_norm": 1.964331865310669, + "learning_rate": 1.8197851391520265e-05, + "loss": 0.7453011274337769, + "step": 77 + }, + { + "epoch": 0.8857345635202271, + "grad_norm": 1.7134088277816772, + "learning_rate": 1.8121222012427666e-05, + "loss": 0.7802327871322632, + "step": 78 + }, + { + "epoch": 0.8970901348474095, + "grad_norm": 1.964368224143982, + "learning_rate": 1.804316565270765e-05, + "loss": 0.8367159366607666, + "step": 79 + }, + { + "epoch": 0.9084457061745919, + "grad_norm": 1.9737969636917114, + "learning_rate": 1.796369602764999e-05, + "loss": 0.8365910053253174, + "step": 80 + }, + { + "epoch": 0.9198012775017743, + "grad_norm": 1.963098168373108, + "learning_rate": 1.788282710086942e-05, + "loss": 0.9032438397407532, + "step": 81 + }, + { + "epoch": 0.9311568488289567, + "grad_norm": 1.771173119544983, + "learning_rate": 1.7800573081852124e-05, + "loss": 0.7875261902809143, + "step": 82 + }, + { + "epoch": 0.9425124201561391, + "grad_norm": 1.9102442264556885, + "learning_rate": 1.771694842345894e-05, + "loss": 0.76175856590271, + "step": 83 + }, + { + "epoch": 0.9538679914833215, + "grad_norm": 2.03609299659729, + "learning_rate": 1.7631967819385883e-05, + "loss": 0.8382056951522827, + "step": 84 + }, + { + "epoch": 0.9652235628105039, + "grad_norm": 1.864434003829956, + "learning_rate": 1.7545646201582304e-05, + "loss": 0.785956084728241, + "step": 85 + }, + { + "epoch": 0.9765791341376863, + "grad_norm": 2.0304460525512695, + "learning_rate": 1.7457998737627183e-05, + "loss": 0.9167435169219971, + "step": 86 + }, + { + "epoch": 0.9879347054648687, + "grad_norm": 1.691648006439209, + "learning_rate": 1.7369040828064046e-05, + "loss": 0.7328086495399475, + "step": 87 + }, + { + "epoch": 0.9992902767920511, + "grad_norm": 2.0275533199310303, + "learning_rate": 1.7278788103694944e-05, + "loss": 0.9172717928886414, + "step": 88 + }, + { + "epoch": 1.0, + "grad_norm": 2.6658289432525635, + "learning_rate": 1.7187256422833928e-05, + "loss": 0.11026950925588608, + "step": 89 + }, + { + "epoch": 1.0113555713271825, + "grad_norm": 1.5766886472702026, + "learning_rate": 1.7094461868520625e-05, + "loss": 0.7454363107681274, + "step": 90 + }, + { + "epoch": 1.0227111426543647, + "grad_norm": 1.934645652770996, + "learning_rate": 1.7000420745694256e-05, + "loss": 0.9695757627487183, + "step": 91 + }, + { + "epoch": 1.0340667139815471, + "grad_norm": 1.7055552005767822, + "learning_rate": 1.6905149578328705e-05, + "loss": 0.7050992846488953, + "step": 92 + }, + { + "epoch": 1.0454222853087296, + "grad_norm": 1.5468671321868896, + "learning_rate": 1.6808665106529096e-05, + "loss": 0.6628597974777222, + "step": 93 + }, + { + "epoch": 1.056777856635912, + "grad_norm": 1.80152428150177, + "learning_rate": 1.671098428359037e-05, + "loss": 0.826714277267456, + "step": 94 + }, + { + "epoch": 1.0681334279630943, + "grad_norm": 1.9063488245010376, + "learning_rate": 1.661212427301844e-05, + "loss": 0.9870727062225342, + "step": 95 + }, + { + "epoch": 1.0794889992902768, + "grad_norm": 1.7208439111709595, + "learning_rate": 1.6512102445514376e-05, + "loss": 0.754402220249176, + "step": 96 + }, + { + "epoch": 1.0908445706174592, + "grad_norm": 1.688193440437317, + "learning_rate": 1.64109363759222e-05, + "loss": 0.6640122532844543, + "step": 97 + }, + { + "epoch": 1.1022001419446417, + "grad_norm": 1.8576338291168213, + "learning_rate": 1.630864384014083e-05, + "loss": 0.8575501441955566, + "step": 98 + }, + { + "epoch": 1.113555713271824, + "grad_norm": 1.7428895235061646, + "learning_rate": 1.620524281200062e-05, + "loss": 0.5853228569030762, + "step": 99 + }, + { + "epoch": 1.1249112845990064, + "grad_norm": 1.5940732955932617, + "learning_rate": 1.6100751460105244e-05, + "loss": 0.6318332552909851, + "step": 100 + }, + { + "epoch": 1.1249112845990064, + "eval_loss": 0.7593621015548706, + "eval_runtime": 27.6141, + "eval_samples_per_second": 5.179, + "eval_steps_per_second": 1.304, + "step": 100 + }, + { + "epoch": 1.1362668559261888, + "grad_norm": 1.9089066982269287, + "learning_rate": 1.599518814463925e-05, + "loss": 0.9151409268379211, + "step": 101 + }, + { + "epoch": 1.1476224272533713, + "grad_norm": 1.905922770500183, + "learning_rate": 1.5888571414141997e-05, + "loss": 0.8580778241157532, + "step": 102 + }, + { + "epoch": 1.1589779985805535, + "grad_norm": 1.964565634727478, + "learning_rate": 1.5780920002248484e-05, + "loss": 0.8872693777084351, + "step": 103 + }, + { + "epoch": 1.170333569907736, + "grad_norm": 2.039443254470825, + "learning_rate": 1.5672252824397683e-05, + "loss": 0.7390342950820923, + "step": 104 + }, + { + "epoch": 1.1816891412349184, + "grad_norm": 1.685909390449524, + "learning_rate": 1.556258897450887e-05, + "loss": 0.6857212781906128, + "step": 105 + }, + { + "epoch": 1.1930447125621009, + "grad_norm": 1.9805678129196167, + "learning_rate": 1.5451947721626676e-05, + "loss": 0.685741662979126, + "step": 106 + }, + { + "epoch": 1.204400283889283, + "grad_norm": 1.6393771171569824, + "learning_rate": 1.534034850653528e-05, + "loss": 0.6081647276878357, + "step": 107 + }, + { + "epoch": 1.2157558552164656, + "grad_norm": 1.371028184890747, + "learning_rate": 1.5227810938342493e-05, + "loss": 0.44291260838508606, + "step": 108 + }, + { + "epoch": 1.227111426543648, + "grad_norm": 1.8976811170578003, + "learning_rate": 1.5114354791034225e-05, + "loss": 0.82900071144104, + "step": 109 + }, + { + "epoch": 1.2384669978708303, + "grad_norm": 1.4269609451293945, + "learning_rate": 1.5000000000000002e-05, + "loss": 0.44125357270240784, + "step": 110 + }, + { + "epoch": 1.2498225691980127, + "grad_norm": 1.9113351106643677, + "learning_rate": 1.4884766658530126e-05, + "loss": 0.7995925545692444, + "step": 111 + }, + { + "epoch": 1.2611781405251952, + "grad_norm": 1.9662408828735352, + "learning_rate": 1.4768675014285063e-05, + "loss": 0.8006548881530762, + "step": 112 + }, + { + "epoch": 1.2725337118523776, + "grad_norm": 2.118868350982666, + "learning_rate": 1.465174546573774e-05, + "loss": 1.086897611618042, + "step": 113 + }, + { + "epoch": 1.28388928317956, + "grad_norm": 1.7964507341384888, + "learning_rate": 1.4533998558589319e-05, + "loss": 0.7104189395904541, + "step": 114 + }, + { + "epoch": 1.2952448545067423, + "grad_norm": 1.659604549407959, + "learning_rate": 1.4415454982159121e-05, + "loss": 0.7449539303779602, + "step": 115 + }, + { + "epoch": 1.3066004258339248, + "grad_norm": 1.7559200525283813, + "learning_rate": 1.429613556574928e-05, + "loss": 0.760001540184021, + "step": 116 + }, + { + "epoch": 1.3179559971611072, + "grad_norm": 1.6712889671325684, + "learning_rate": 1.4176061274984858e-05, + "loss": 0.5844006538391113, + "step": 117 + }, + { + "epoch": 1.3293115684882895, + "grad_norm": 1.8857864141464233, + "learning_rate": 1.405525320812994e-05, + "loss": 0.799013614654541, + "step": 118 + }, + { + "epoch": 1.340667139815472, + "grad_norm": 1.9506901502609253, + "learning_rate": 1.3933732592380485e-05, + "loss": 0.8254635334014893, + "step": 119 + }, + { + "epoch": 1.3520227111426544, + "grad_norm": 1.556196689605713, + "learning_rate": 1.3811520780134471e-05, + "loss": 0.6055241227149963, + "step": 120 + }, + { + "epoch": 1.3633782824698368, + "grad_norm": 1.5888230800628662, + "learning_rate": 1.3688639245240078e-05, + "loss": 0.641814112663269, + "step": 121 + }, + { + "epoch": 1.3747338537970193, + "grad_norm": 1.740252137184143, + "learning_rate": 1.3565109579222511e-05, + "loss": 0.746912956237793, + "step": 122 + }, + { + "epoch": 1.3860894251242015, + "grad_norm": 2.0926947593688965, + "learning_rate": 1.3440953487490145e-05, + "loss": 0.8887531161308289, + "step": 123 + }, + { + "epoch": 1.397444996451384, + "grad_norm": 1.7420989274978638, + "learning_rate": 1.331619278552068e-05, + "loss": 0.6902853846549988, + "step": 124 + }, + { + "epoch": 1.4088005677785664, + "grad_norm": 1.8094593286514282, + "learning_rate": 1.3190849395027926e-05, + "loss": 0.7649242877960205, + "step": 125 + }, + { + "epoch": 1.4201561391057487, + "grad_norm": 1.4437446594238281, + "learning_rate": 1.306494534010995e-05, + "loss": 0.5692201256752014, + "step": 126 + }, + { + "epoch": 1.4315117104329311, + "grad_norm": 1.3073190450668335, + "learning_rate": 1.2938502743379212e-05, + "loss": 0.4875521957874298, + "step": 127 + }, + { + "epoch": 1.4428672817601136, + "grad_norm": 1.6500370502471924, + "learning_rate": 1.2811543822075396e-05, + "loss": 0.6286922097206116, + "step": 128 + }, + { + "epoch": 1.454222853087296, + "grad_norm": 1.4302570819854736, + "learning_rate": 1.2684090884161636e-05, + "loss": 0.48701754212379456, + "step": 129 + }, + { + "epoch": 1.4655784244144783, + "grad_norm": 1.7269134521484375, + "learning_rate": 1.2556166324404747e-05, + "loss": 0.6981077790260315, + "step": 130 + }, + { + "epoch": 1.4769339957416607, + "grad_norm": 1.5457602739334106, + "learning_rate": 1.242779262044028e-05, + "loss": 0.48003125190734863, + "step": 131 + }, + { + "epoch": 1.4882895670688432, + "grad_norm": 1.8600218296051025, + "learning_rate": 1.2298992328822937e-05, + "loss": 0.7610920667648315, + "step": 132 + }, + { + "epoch": 1.4996451383960254, + "grad_norm": 1.3750872611999512, + "learning_rate": 1.2169788081063181e-05, + "loss": 0.4584071934223175, + "step": 133 + }, + { + "epoch": 1.5110007097232079, + "grad_norm": 1.5970045328140259, + "learning_rate": 1.2040202579650649e-05, + "loss": 0.6092408895492554, + "step": 134 + }, + { + "epoch": 1.5223562810503903, + "grad_norm": 1.8978732824325562, + "learning_rate": 1.1910258594065079e-05, + "loss": 0.8766818642616272, + "step": 135 + }, + { + "epoch": 1.5337118523775728, + "grad_norm": 1.7095293998718262, + "learning_rate": 1.1779978956775507e-05, + "loss": 0.629298985004425, + "step": 136 + }, + { + "epoch": 1.5450674237047552, + "grad_norm": 1.8608379364013672, + "learning_rate": 1.1649386559228342e-05, + "loss": 0.6764330267906189, + "step": 137 + }, + { + "epoch": 1.5564229950319377, + "grad_norm": 1.760704517364502, + "learning_rate": 1.1518504347825146e-05, + "loss": 0.7110517621040344, + "step": 138 + }, + { + "epoch": 1.56777856635912, + "grad_norm": 1.937847375869751, + "learning_rate": 1.1387355319890685e-05, + "loss": 0.7304476499557495, + "step": 139 + }, + { + "epoch": 1.5791341376863024, + "grad_norm": 1.5163425207138062, + "learning_rate": 1.1255962519632082e-05, + "loss": 0.6055701971054077, + "step": 140 + }, + { + "epoch": 1.5904897090134846, + "grad_norm": 1.6075628995895386, + "learning_rate": 1.1124349034089724e-05, + "loss": 0.5708965063095093, + "step": 141 + }, + { + "epoch": 1.601845280340667, + "grad_norm": 1.7377432584762573, + "learning_rate": 1.0992537989080618e-05, + "loss": 0.672942042350769, + "step": 142 + }, + { + "epoch": 1.6132008516678495, + "grad_norm": 2.052077293395996, + "learning_rate": 1.086055254513497e-05, + "loss": 0.7728007435798645, + "step": 143 + }, + { + "epoch": 1.624556422995032, + "grad_norm": 1.5325052738189697, + "learning_rate": 1.0728415893426636e-05, + "loss": 0.5753393769264221, + "step": 144 + }, + { + "epoch": 1.6359119943222145, + "grad_norm": 2.0628015995025635, + "learning_rate": 1.05961512516982e-05, + "loss": 0.7965511083602905, + "step": 145 + }, + { + "epoch": 1.6472675656493967, + "grad_norm": 1.5534552335739136, + "learning_rate": 1.0463781860181385e-05, + "loss": 0.4485720694065094, + "step": 146 + }, + { + "epoch": 1.6586231369765791, + "grad_norm": 2.0554072856903076, + "learning_rate": 1.033133097751351e-05, + "loss": 0.87266606092453, + "step": 147 + }, + { + "epoch": 1.6699787083037614, + "grad_norm": 1.702256441116333, + "learning_rate": 1.0198821876650702e-05, + "loss": 0.5705003142356873, + "step": 148 + }, + { + "epoch": 1.6813342796309438, + "grad_norm": 1.5394976139068604, + "learning_rate": 1.0066277840778626e-05, + "loss": 0.4785357117652893, + "step": 149 + }, + { + "epoch": 1.6926898509581263, + "grad_norm": 1.7153429985046387, + "learning_rate": 9.933722159221375e-06, + "loss": 0.7122579216957092, + "step": 150 + }, + { + "epoch": 1.6926898509581263, + "eval_loss": 0.7425792217254639, + "eval_runtime": 27.5719, + "eval_samples_per_second": 5.186, + "eval_steps_per_second": 1.306, + "step": 150 + }, + { + "epoch": 1.7040454222853088, + "grad_norm": 1.7121310234069824, + "learning_rate": 9.801178123349298e-06, + "loss": 0.6155564188957214, + "step": 151 + }, + { + "epoch": 1.7154009936124912, + "grad_norm": 1.8881406784057617, + "learning_rate": 9.668669022486495e-06, + "loss": 0.7582035064697266, + "step": 152 + }, + { + "epoch": 1.7267565649396737, + "grad_norm": 1.9322803020477295, + "learning_rate": 9.536218139818615e-06, + "loss": 0.8061675429344177, + "step": 153 + }, + { + "epoch": 1.738112136266856, + "grad_norm": 1.5622960329055786, + "learning_rate": 9.403848748301802e-06, + "loss": 0.5297854542732239, + "step": 154 + }, + { + "epoch": 1.7494677075940384, + "grad_norm": 1.8679121732711792, + "learning_rate": 9.271584106573364e-06, + "loss": 0.6446192264556885, + "step": 155 + }, + { + "epoch": 1.7608232789212206, + "grad_norm": 1.65900456905365, + "learning_rate": 9.139447454865034e-06, + "loss": 0.6376427412033081, + "step": 156 + }, + { + "epoch": 1.772178850248403, + "grad_norm": 2.008634328842163, + "learning_rate": 9.007462010919387e-06, + "loss": 0.8298434615135193, + "step": 157 + }, + { + "epoch": 1.7835344215755855, + "grad_norm": 1.817112684249878, + "learning_rate": 8.87565096591028e-06, + "loss": 0.6945917010307312, + "step": 158 + }, + { + "epoch": 1.794889992902768, + "grad_norm": 2.0954995155334473, + "learning_rate": 8.744037480367922e-06, + "loss": 0.922713577747345, + "step": 159 + }, + { + "epoch": 1.8062455642299504, + "grad_norm": 1.8708635568618774, + "learning_rate": 8.61264468010932e-06, + "loss": 0.7695997953414917, + "step": 160 + }, + { + "epoch": 1.8176011355571329, + "grad_norm": 2.0717625617980957, + "learning_rate": 8.481495652174859e-06, + "loss": 0.9634745121002197, + "step": 161 + }, + { + "epoch": 1.828956706884315, + "grad_norm": 1.4181921482086182, + "learning_rate": 8.350613440771661e-06, + "loss": 0.41311249136924744, + "step": 162 + }, + { + "epoch": 1.8403122782114976, + "grad_norm": 1.5503257513046265, + "learning_rate": 8.2200210432245e-06, + "loss": 0.6834385395050049, + "step": 163 + }, + { + "epoch": 1.8516678495386798, + "grad_norm": 1.767377257347107, + "learning_rate": 8.089741405934923e-06, + "loss": 0.7185457944869995, + "step": 164 + }, + { + "epoch": 1.8630234208658623, + "grad_norm": 1.8889371156692505, + "learning_rate": 7.959797420349356e-06, + "loss": 0.9002049565315247, + "step": 165 + }, + { + "epoch": 1.8743789921930447, + "grad_norm": 2.01930832862854, + "learning_rate": 7.83021191893682e-06, + "loss": 0.6417741775512695, + "step": 166 + }, + { + "epoch": 1.8857345635202272, + "grad_norm": 1.8323336839675903, + "learning_rate": 7.701007671177066e-06, + "loss": 0.8583472371101379, + "step": 167 + }, + { + "epoch": 1.8970901348474096, + "grad_norm": 1.3641561269760132, + "learning_rate": 7.572207379559722e-06, + "loss": 0.5814041495323181, + "step": 168 + }, + { + "epoch": 1.9084457061745919, + "grad_norm": 1.6025844812393188, + "learning_rate": 7.443833675595254e-06, + "loss": 0.5391830205917358, + "step": 169 + }, + { + "epoch": 1.9198012775017743, + "grad_norm": 1.523732304573059, + "learning_rate": 7.315909115838367e-06, + "loss": 0.6024059057235718, + "step": 170 + }, + { + "epoch": 1.9311568488289566, + "grad_norm": 1.6438531875610352, + "learning_rate": 7.1884561779246055e-06, + "loss": 0.5504345297813416, + "step": 171 + }, + { + "epoch": 1.942512420156139, + "grad_norm": 1.884101152420044, + "learning_rate": 7.061497256620793e-06, + "loss": 0.8089767694473267, + "step": 172 + }, + { + "epoch": 1.9538679914833215, + "grad_norm": 1.705494999885559, + "learning_rate": 6.935054659890053e-06, + "loss": 0.6709920763969421, + "step": 173 + }, + { + "epoch": 1.965223562810504, + "grad_norm": 1.8286340236663818, + "learning_rate": 6.809150604972079e-06, + "loss": 0.9714908003807068, + "step": 174 + }, + { + "epoch": 1.9765791341376864, + "grad_norm": 1.6881917715072632, + "learning_rate": 6.683807214479323e-06, + "loss": 0.6124534606933594, + "step": 175 + }, + { + "epoch": 1.9879347054648688, + "grad_norm": 2.147641181945801, + "learning_rate": 6.559046512509859e-06, + "loss": 0.9034600257873535, + "step": 176 + }, + { + "epoch": 1.999290276792051, + "grad_norm": 1.7281945943832397, + "learning_rate": 6.434890420777491e-06, + "loss": 0.7872213125228882, + "step": 177 + }, + { + "epoch": 2.0, + "grad_norm": 4.5360822677612305, + "learning_rate": 6.311360754759923e-06, + "loss": 0.2226628065109253, + "step": 178 + }, + { + "epoch": 2.0113555713271825, + "grad_norm": 1.6504145860671997, + "learning_rate": 6.188479219865529e-06, + "loss": 0.6821281313896179, + "step": 179 + }, + { + "epoch": 2.022711142654365, + "grad_norm": 1.4790889024734497, + "learning_rate": 6.06626740761952e-06, + "loss": 0.5527986288070679, + "step": 180 + }, + { + "epoch": 2.0340667139815474, + "grad_norm": 1.8767532110214233, + "learning_rate": 5.944746791870062e-06, + "loss": 0.7406971454620361, + "step": 181 + }, + { + "epoch": 2.0454222853087294, + "grad_norm": 1.6067395210266113, + "learning_rate": 5.823938725015148e-06, + "loss": 0.5897183418273926, + "step": 182 + }, + { + "epoch": 2.056777856635912, + "grad_norm": 1.6543129682540894, + "learning_rate": 5.703864434250721e-06, + "loss": 0.5693725943565369, + "step": 183 + }, + { + "epoch": 2.0681334279630943, + "grad_norm": 1.763333797454834, + "learning_rate": 5.584545017840886e-06, + "loss": 0.6083229184150696, + "step": 184 + }, + { + "epoch": 2.0794889992902768, + "grad_norm": 1.2259186506271362, + "learning_rate": 5.4660014414106825e-06, + "loss": 0.3644542098045349, + "step": 185 + }, + { + "epoch": 2.090844570617459, + "grad_norm": 1.6024012565612793, + "learning_rate": 5.348254534262262e-06, + "loss": 0.607455313205719, + "step": 186 + }, + { + "epoch": 2.1022001419446417, + "grad_norm": 1.5699106454849243, + "learning_rate": 5.231324985714942e-06, + "loss": 0.4221411645412445, + "step": 187 + }, + { + "epoch": 2.113555713271824, + "grad_norm": 1.439180850982666, + "learning_rate": 5.1152333414698774e-06, + "loss": 0.4416266679763794, + "step": 188 + }, + { + "epoch": 2.1249112845990066, + "grad_norm": 1.8221555948257446, + "learning_rate": 5.000000000000003e-06, + "loss": 0.5936031341552734, + "step": 189 + }, + { + "epoch": 2.1362668559261886, + "grad_norm": 1.888954520225525, + "learning_rate": 4.885645208965779e-06, + "loss": 0.7918609976768494, + "step": 190 + }, + { + "epoch": 2.147622427253371, + "grad_norm": 1.6411021947860718, + "learning_rate": 4.772189061657511e-06, + "loss": 0.5752283334732056, + "step": 191 + }, + { + "epoch": 2.1589779985805535, + "grad_norm": 1.6003109216690063, + "learning_rate": 4.659651493464721e-06, + "loss": 0.6083833575248718, + "step": 192 + }, + { + "epoch": 2.170333569907736, + "grad_norm": 1.657971739768982, + "learning_rate": 4.548052278373327e-06, + "loss": 0.6448208093643188, + "step": 193 + }, + { + "epoch": 2.1816891412349184, + "grad_norm": 1.9829140901565552, + "learning_rate": 4.437411025491131e-06, + "loss": 0.6960307359695435, + "step": 194 + }, + { + "epoch": 2.193044712562101, + "grad_norm": 1.8921784162521362, + "learning_rate": 4.327747175602321e-06, + "loss": 0.731373131275177, + "step": 195 + }, + { + "epoch": 2.2044002838892833, + "grad_norm": 1.7255470752716064, + "learning_rate": 4.219079997751515e-06, + "loss": 0.5975107550621033, + "step": 196 + }, + { + "epoch": 2.215755855216466, + "grad_norm": 1.5999464988708496, + "learning_rate": 4.111428585858005e-06, + "loss": 0.5760666131973267, + "step": 197 + }, + { + "epoch": 2.227111426543648, + "grad_norm": 1.240045428276062, + "learning_rate": 4.0048118553607485e-06, + "loss": 0.39564967155456543, + "step": 198 + }, + { + "epoch": 2.2384669978708303, + "grad_norm": 1.9376376867294312, + "learning_rate": 3.899248539894756e-06, + "loss": 0.6683239936828613, + "step": 199 + }, + { + "epoch": 2.2498225691980127, + "grad_norm": 1.8060548305511475, + "learning_rate": 3.794757187999386e-06, + "loss": 0.7160100936889648, + "step": 200 + }, + { + "epoch": 2.2498225691980127, + "eval_loss": 0.7327238321304321, + "eval_runtime": 27.6122, + "eval_samples_per_second": 5.179, + "eval_steps_per_second": 1.304, + "step": 200 + }, + { + "epoch": 2.261178140525195, + "grad_norm": 1.5847166776657104, + "learning_rate": 3.6913561598591775e-06, + "loss": 0.5229685306549072, + "step": 201 + }, + { + "epoch": 2.2725337118523776, + "grad_norm": 1.6213287115097046, + "learning_rate": 3.589063624077802e-06, + "loss": 0.5970481634140015, + "step": 202 + }, + { + "epoch": 2.28388928317956, + "grad_norm": 1.818298578262329, + "learning_rate": 3.4878975544856285e-06, + "loss": 0.7519898414611816, + "step": 203 + }, + { + "epoch": 2.2952448545067425, + "grad_norm": 1.6277714967727661, + "learning_rate": 3.387875726981563e-06, + "loss": 0.5007752180099487, + "step": 204 + }, + { + "epoch": 2.306600425833925, + "grad_norm": 1.6472209692001343, + "learning_rate": 3.2890157164096315e-06, + "loss": 0.6382125616073608, + "step": 205 + }, + { + "epoch": 2.317955997161107, + "grad_norm": 1.821578025817871, + "learning_rate": 3.1913348934709076e-06, + "loss": 0.7000253200531006, + "step": 206 + }, + { + "epoch": 2.3293115684882895, + "grad_norm": 1.8459348678588867, + "learning_rate": 3.094850421671295e-06, + "loss": 0.5165408253669739, + "step": 207 + }, + { + "epoch": 2.340667139815472, + "grad_norm": 2.36333966255188, + "learning_rate": 2.999579254305748e-06, + "loss": 0.8205851912498474, + "step": 208 + }, + { + "epoch": 2.3520227111426544, + "grad_norm": 1.6719001531600952, + "learning_rate": 2.905538131479376e-06, + "loss": 0.5079140663146973, + "step": 209 + }, + { + "epoch": 2.363378282469837, + "grad_norm": 1.5651746988296509, + "learning_rate": 2.812743577166075e-06, + "loss": 0.4731737971305847, + "step": 210 + }, + { + "epoch": 2.3747338537970193, + "grad_norm": 2.142404317855835, + "learning_rate": 2.721211896305059e-06, + "loss": 0.842810869216919, + "step": 211 + }, + { + "epoch": 2.3860894251242017, + "grad_norm": 1.796759843826294, + "learning_rate": 2.6309591719359563e-06, + "loss": 0.7187446355819702, + "step": 212 + }, + { + "epoch": 2.3974449964513838, + "grad_norm": 1.492469310760498, + "learning_rate": 2.542001262372821e-06, + "loss": 0.5724196434020996, + "step": 213 + }, + { + "epoch": 2.408800567778566, + "grad_norm": 1.905091643333435, + "learning_rate": 2.454353798417698e-06, + "loss": 0.8085651397705078, + "step": 214 + }, + { + "epoch": 2.4201561391057487, + "grad_norm": 1.7704670429229736, + "learning_rate": 2.3680321806141182e-06, + "loss": 0.6132161021232605, + "step": 215 + }, + { + "epoch": 2.431511710432931, + "grad_norm": 1.8361713886260986, + "learning_rate": 2.283051576541062e-06, + "loss": 0.7117694616317749, + "step": 216 + }, + { + "epoch": 2.4428672817601136, + "grad_norm": 1.6990126371383667, + "learning_rate": 2.19942691814788e-06, + "loss": 0.6097843050956726, + "step": 217 + }, + { + "epoch": 2.454222853087296, + "grad_norm": 1.7435643672943115, + "learning_rate": 2.1171728991305797e-06, + "loss": 0.5934941172599792, + "step": 218 + }, + { + "epoch": 2.4655784244144785, + "grad_norm": 1.9126805067062378, + "learning_rate": 2.0363039723500155e-06, + "loss": 0.704249918460846, + "step": 219 + }, + { + "epoch": 2.4769339957416605, + "grad_norm": 1.7842662334442139, + "learning_rate": 1.9568343472923524e-06, + "loss": 0.6474202871322632, + "step": 220 + }, + { + "epoch": 2.488289567068843, + "grad_norm": 1.7116307020187378, + "learning_rate": 1.8787779875723389e-06, + "loss": 0.5358954668045044, + "step": 221 + }, + { + "epoch": 2.4996451383960254, + "grad_norm": 1.9682855606079102, + "learning_rate": 1.8021486084797368e-06, + "loss": 0.7930698990821838, + "step": 222 + }, + { + "epoch": 2.511000709723208, + "grad_norm": 1.4548183679580688, + "learning_rate": 1.7269596745694295e-06, + "loss": 0.4583667814731598, + "step": 223 + }, + { + "epoch": 2.5223562810503903, + "grad_norm": 2.000056505203247, + "learning_rate": 1.6532243972955397e-06, + "loss": 0.8167339563369751, + "step": 224 + }, + { + "epoch": 2.533711852377573, + "grad_norm": 2.187757730484009, + "learning_rate": 1.580955732690065e-06, + "loss": 0.8731608390808105, + "step": 225 + }, + { + "epoch": 2.5450674237047552, + "grad_norm": 2.552974224090576, + "learning_rate": 1.5101663790863597e-06, + "loss": 0.9876814484596252, + "step": 226 + }, + { + "epoch": 2.5564229950319377, + "grad_norm": 2.047971725463867, + "learning_rate": 1.4408687748879157e-06, + "loss": 0.7313542366027832, + "step": 227 + }, + { + "epoch": 2.56777856635912, + "grad_norm": 2.023643732070923, + "learning_rate": 1.3730750963828033e-06, + "loss": 0.7768765091896057, + "step": 228 + }, + { + "epoch": 2.579134137686302, + "grad_norm": 1.7678884267807007, + "learning_rate": 1.3067972556041753e-06, + "loss": 0.5483944416046143, + "step": 229 + }, + { + "epoch": 2.5904897090134846, + "grad_norm": 1.57295560836792, + "learning_rate": 1.2420468982372158e-06, + "loss": 0.5286450386047363, + "step": 230 + }, + { + "epoch": 2.601845280340667, + "grad_norm": 2.199338912963867, + "learning_rate": 1.1788354015728543e-06, + "loss": 0.7952181100845337, + "step": 231 + }, + { + "epoch": 2.6132008516678495, + "grad_norm": 2.037968635559082, + "learning_rate": 1.1171738725086833e-06, + "loss": 0.8746033906936646, + "step": 232 + }, + { + "epoch": 2.624556422995032, + "grad_norm": 1.6114685535430908, + "learning_rate": 1.0570731455973415e-06, + "loss": 0.5900834202766418, + "step": 233 + }, + { + "epoch": 2.6359119943222145, + "grad_norm": 1.8988250494003296, + "learning_rate": 9.985437811427934e-07, + "loss": 0.7454937696456909, + "step": 234 + }, + { + "epoch": 2.647267565649397, + "grad_norm": 1.6500483751296997, + "learning_rate": 9.415960633447674e-07, + "loss": 0.5034666657447815, + "step": 235 + }, + { + "epoch": 2.658623136976579, + "grad_norm": 1.9361827373504639, + "learning_rate": 8.862399984917214e-07, + "loss": 0.651042103767395, + "step": 236 + }, + { + "epoch": 2.6699787083037614, + "grad_norm": 1.9240092039108276, + "learning_rate": 8.3248531320263e-07, + "loss": 0.8603060245513916, + "step": 237 + }, + { + "epoch": 2.681334279630944, + "grad_norm": 1.9688630104064941, + "learning_rate": 7.803414527179343e-07, + "loss": 0.8090041875839233, + "step": 238 + }, + { + "epoch": 2.6926898509581263, + "grad_norm": 1.7253657579421997, + "learning_rate": 7.298175792398976e-07, + "loss": 0.5564637184143066, + "step": 239 + }, + { + "epoch": 2.7040454222853088, + "grad_norm": 1.871412754058838, + "learning_rate": 6.809225703227352e-07, + "loss": 0.7740951776504517, + "step": 240 + }, + { + "epoch": 2.715400993612491, + "grad_norm": 1.8274517059326172, + "learning_rate": 6.336650173127224e-07, + "loss": 0.6926577091217041, + "step": 241 + }, + { + "epoch": 2.7267565649396737, + "grad_norm": 2.128312349319458, + "learning_rate": 5.880532238386161e-07, + "loss": 0.8623908758163452, + "step": 242 + }, + { + "epoch": 2.7381121362668557, + "grad_norm": 1.9063048362731934, + "learning_rate": 5.440952043526215e-07, + "loss": 0.642934262752533, + "step": 243 + }, + { + "epoch": 2.7494677075940386, + "grad_norm": 1.9280798435211182, + "learning_rate": 5.017986827221733e-07, + "loss": 0.5877834558486938, + "step": 244 + }, + { + "epoch": 2.7608232789212206, + "grad_norm": 1.7953683137893677, + "learning_rate": 4.61171090872774e-07, + "loss": 0.6689532995223999, + "step": 245 + }, + { + "epoch": 2.772178850248403, + "grad_norm": 2.076279640197754, + "learning_rate": 4.222195674821239e-07, + "loss": 0.8235094547271729, + "step": 246 + }, + { + "epoch": 2.7835344215755855, + "grad_norm": 2.2308530807495117, + "learning_rate": 3.8495095672579584e-07, + "loss": 0.8819231986999512, + "step": 247 + }, + { + "epoch": 2.794889992902768, + "grad_norm": 1.5626147985458374, + "learning_rate": 3.493718070746299e-07, + "loss": 0.45997241139411926, + "step": 248 + }, + { + "epoch": 2.8062455642299504, + "grad_norm": 1.80852472782135, + "learning_rate": 3.154883701441136e-07, + "loss": 0.5900078415870667, + "step": 249 + }, + { + "epoch": 2.817601135557133, + "grad_norm": 1.591232180595398, + "learning_rate": 2.8330659959589944e-07, + "loss": 0.4933575987815857, + "step": 250 + }, + { + "epoch": 2.817601135557133, + "eval_loss": 0.7293056845664978, + "eval_runtime": 27.6187, + "eval_samples_per_second": 5.178, + "eval_steps_per_second": 1.303, + "step": 250 + } + ], + "logging_steps": 1, + "max_steps": 267, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.782841153099264e+16, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/lora_adapter/checkpoint-250/training_args.bin b/lora_adapter/checkpoint-250/training_args.bin new file mode 100644 index 0000000..35b4cdb --- /dev/null +++ b/lora_adapter/checkpoint-250/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ebafba8635254798c46cfc655a273a08f5a7d82a00226b90c1333350daf6f6b +size 5713 diff --git a/lora_adapter/checkpoint-267/README.md b/lora_adapter/checkpoint-267/README.md new file mode 100644 index 0000000..a4c765d --- /dev/null +++ b/lora_adapter/checkpoint-267/README.md @@ -0,0 +1,210 @@ +--- +base_model: unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit +- lora +- sft +- transformers +- trl +- unsloth +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/lora_adapter/checkpoint-267/adapter_config.json b/lora_adapter/checkpoint-267/adapter_config.json new file mode 100644 index 0000000..6e70f33 --- /dev/null +++ b/lora_adapter/checkpoint-267/adapter_config.json @@ -0,0 +1,52 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": { + "base_model_class": "Qwen2ForCausalLM", + "parent_library": "transformers.models.qwen2.modeling_qwen2", + "unsloth_fixed": true + }, + "base_model_name_or_path": "unsloth/qwen2.5-1.5b-instruct-unsloth-bnb-4bit", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.0, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "up_proj", + "gate_proj", + "q_proj", + "v_proj", + "down_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": true +} \ No newline at end of file diff --git a/lora_adapter/checkpoint-267/adapter_model.safetensors b/lora_adapter/checkpoint-267/adapter_model.safetensors new file mode 100644 index 0000000..5c49ce4 --- /dev/null +++ b/lora_adapter/checkpoint-267/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:314dddcefa71a50e27cf6fb56643faa9519c6213f22380d13059cb47fde754fb +size 295488936 diff --git a/lora_adapter/checkpoint-267/chat_template.jinja b/lora_adapter/checkpoint-267/chat_template.jinja new file mode 100644 index 0000000..642e597 --- /dev/null +++ b/lora_adapter/checkpoint-267/chat_template.jinja @@ -0,0 +1,53 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/lora_adapter/checkpoint-267/optimizer.pt b/lora_adapter/checkpoint-267/optimizer.pt new file mode 100644 index 0000000..9c1c67b --- /dev/null +++ b/lora_adapter/checkpoint-267/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91360943acb3e372773e0520b6d85eb5d2d99da45923ceaf86d5736363759899 +size 150491717 diff --git a/lora_adapter/checkpoint-267/rng_state.pth b/lora_adapter/checkpoint-267/rng_state.pth new file mode 100644 index 0000000..eed58b8 --- /dev/null +++ b/lora_adapter/checkpoint-267/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:363c5df1543d2c82b2f13164f35bdd0367ceb32e7fa1b2f67c19df073a08b17b +size 14645 diff --git a/lora_adapter/checkpoint-267/scheduler.pt b/lora_adapter/checkpoint-267/scheduler.pt new file mode 100644 index 0000000..82d3a71 --- /dev/null +++ b/lora_adapter/checkpoint-267/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e7b44e32b782ecb105c0b1658809aa02dad672d4b027dd46cdff3d2842425c5 +size 1465 diff --git a/lora_adapter/checkpoint-267/tokenizer.json b/lora_adapter/checkpoint-267/tokenizer.json new file mode 100644 index 0000000..5340d81 --- /dev/null +++ b/lora_adapter/checkpoint-267/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd5948af71b4f56cf697f7580814c7ce8b80595ef985544efcacf716126a2e31 +size 11422356 diff --git a/lora_adapter/checkpoint-267/tokenizer_config.json b/lora_adapter/checkpoint-267/tokenizer_config.json new file mode 100644 index 0000000..d3dac53 --- /dev/null +++ b/lora_adapter/checkpoint-267/tokenizer_config.json @@ -0,0 +1,201 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 32768, + "pad_token": "<|PAD_TOKEN|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151657": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151658": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151665": { + "content": "<|PAD_TOKEN|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + } +} diff --git a/lora_adapter/checkpoint-267/trainer_state.json b/lora_adapter/checkpoint-267/trainer_state.json new file mode 100644 index 0000000..515633e --- /dev/null +++ b/lora_adapter/checkpoint-267/trainer_state.json @@ -0,0 +1,1951 @@ +{ + "best_global_step": 250, + "best_metric": 0.7293056845664978, + "best_model_checkpoint": "./qwen_unified_finetuned/checkpoint-250", + "epoch": 3.0, + "eval_steps": 50, + "global_step": 267, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0113555713271824, + "grad_norm": 3.916877031326294, + "learning_rate": 0.0, + "loss": 1.048294186592102, + "step": 1 + }, + { + "epoch": 0.0227111426543648, + "grad_norm": 4.441281318664551, + "learning_rate": 6.666666666666667e-07, + "loss": 0.8231856822967529, + "step": 2 + }, + { + "epoch": 0.034066713981547196, + "grad_norm": 4.950929641723633, + "learning_rate": 1.3333333333333334e-06, + "loss": 1.31972074508667, + "step": 3 + }, + { + "epoch": 0.0454222853087296, + "grad_norm": 4.647597789764404, + "learning_rate": 2.0000000000000003e-06, + "loss": 1.1833959817886353, + "step": 4 + }, + { + "epoch": 0.05677785663591199, + "grad_norm": 4.787125110626221, + "learning_rate": 2.666666666666667e-06, + "loss": 1.5079869031906128, + "step": 5 + }, + { + "epoch": 0.06813342796309439, + "grad_norm": 4.424062252044678, + "learning_rate": 3.3333333333333333e-06, + "loss": 1.0498075485229492, + "step": 6 + }, + { + "epoch": 0.0794889992902768, + "grad_norm": 4.251436710357666, + "learning_rate": 4.000000000000001e-06, + "loss": 1.0172538757324219, + "step": 7 + }, + { + "epoch": 0.0908445706174592, + "grad_norm": 3.5152761936187744, + "learning_rate": 4.666666666666667e-06, + "loss": 1.1070594787597656, + "step": 8 + }, + { + "epoch": 0.1022001419446416, + "grad_norm": 3.2286136150360107, + "learning_rate": 5.333333333333334e-06, + "loss": 0.8639000058174133, + "step": 9 + }, + { + "epoch": 0.11355571327182398, + "grad_norm": 2.9262876510620117, + "learning_rate": 6e-06, + "loss": 0.927219033241272, + "step": 10 + }, + { + "epoch": 0.12491128459900638, + "grad_norm": 3.439363956451416, + "learning_rate": 6.666666666666667e-06, + "loss": 1.2999275922775269, + "step": 11 + }, + { + "epoch": 0.13626685592618878, + "grad_norm": 2.8811707496643066, + "learning_rate": 7.333333333333333e-06, + "loss": 1.0791280269622803, + "step": 12 + }, + { + "epoch": 0.14762242725337119, + "grad_norm": 2.5855603218078613, + "learning_rate": 8.000000000000001e-06, + "loss": 1.0025461912155151, + "step": 13 + }, + { + "epoch": 0.1589779985805536, + "grad_norm": 2.93908953666687, + "learning_rate": 8.666666666666668e-06, + "loss": 1.2033611536026, + "step": 14 + }, + { + "epoch": 0.170333569907736, + "grad_norm": 2.1815803050994873, + "learning_rate": 9.333333333333334e-06, + "loss": 0.865071177482605, + "step": 15 + }, + { + "epoch": 0.1816891412349184, + "grad_norm": 2.5811753273010254, + "learning_rate": 1e-05, + "loss": 0.9821955561637878, + "step": 16 + }, + { + "epoch": 0.1930447125621008, + "grad_norm": 2.2700631618499756, + "learning_rate": 1.0666666666666667e-05, + "loss": 0.943819522857666, + "step": 17 + }, + { + "epoch": 0.2044002838892832, + "grad_norm": 2.234851360321045, + "learning_rate": 1.1333333333333334e-05, + "loss": 0.8833191990852356, + "step": 18 + }, + { + "epoch": 0.2157558552164656, + "grad_norm": 2.1426126956939697, + "learning_rate": 1.2e-05, + "loss": 0.8689224720001221, + "step": 19 + }, + { + "epoch": 0.22711142654364797, + "grad_norm": 2.0993337631225586, + "learning_rate": 1.2666666666666667e-05, + "loss": 0.8528774380683899, + "step": 20 + }, + { + "epoch": 0.23846699787083037, + "grad_norm": 1.9766696691513062, + "learning_rate": 1.3333333333333333e-05, + "loss": 0.889632523059845, + "step": 21 + }, + { + "epoch": 0.24982256919801277, + "grad_norm": 2.3761353492736816, + "learning_rate": 1.4e-05, + "loss": 0.9641993045806885, + "step": 22 + }, + { + "epoch": 0.26117814052519517, + "grad_norm": 2.226921558380127, + "learning_rate": 1.4666666666666666e-05, + "loss": 1.0349667072296143, + "step": 23 + }, + { + "epoch": 0.27253371185237757, + "grad_norm": 1.9597960710525513, + "learning_rate": 1.5333333333333334e-05, + "loss": 0.8142374157905579, + "step": 24 + }, + { + "epoch": 0.28388928317955997, + "grad_norm": 1.769181728363037, + "learning_rate": 1.6000000000000003e-05, + "loss": 0.6785438656806946, + "step": 25 + }, + { + "epoch": 0.29524485450674237, + "grad_norm": 2.046727418899536, + "learning_rate": 1.6666666666666667e-05, + "loss": 0.9442430734634399, + "step": 26 + }, + { + "epoch": 0.3066004258339248, + "grad_norm": 2.2059895992279053, + "learning_rate": 1.7333333333333336e-05, + "loss": 1.0432536602020264, + "step": 27 + }, + { + "epoch": 0.3179559971611072, + "grad_norm": 2.3628499507904053, + "learning_rate": 1.8e-05, + "loss": 1.0692076683044434, + "step": 28 + }, + { + "epoch": 0.3293115684882896, + "grad_norm": 2.2492411136627197, + "learning_rate": 1.866666666666667e-05, + "loss": 1.1659228801727295, + "step": 29 + }, + { + "epoch": 0.340667139815472, + "grad_norm": 2.217177152633667, + "learning_rate": 1.9333333333333333e-05, + "loss": 0.706802487373352, + "step": 30 + }, + { + "epoch": 0.3520227111426544, + "grad_norm": 1.6823749542236328, + "learning_rate": 2e-05, + "loss": 0.7545626163482666, + "step": 31 + }, + { + "epoch": 0.3633782824698368, + "grad_norm": 2.0650510787963867, + "learning_rate": 1.9999121449564347e-05, + "loss": 0.8590264320373535, + "step": 32 + }, + { + "epoch": 0.3747338537970192, + "grad_norm": 1.6296216249465942, + "learning_rate": 1.9996485952627554e-05, + "loss": 0.7331938743591309, + "step": 33 + }, + { + "epoch": 0.3860894251242016, + "grad_norm": 1.7522505521774292, + "learning_rate": 1.999209397227302e-05, + "loss": 0.7665808796882629, + "step": 34 + }, + { + "epoch": 0.397444996451384, + "grad_norm": 2.035996198654175, + "learning_rate": 1.9985946280215996e-05, + "loss": 0.9990826845169067, + "step": 35 + }, + { + "epoch": 0.4088005677785664, + "grad_norm": 1.993740200996399, + "learning_rate": 1.997804395666799e-05, + "loss": 0.8873282074928284, + "step": 36 + }, + { + "epoch": 0.4201561391057488, + "grad_norm": 1.9349602460861206, + "learning_rate": 1.996838839014696e-05, + "loss": 0.8252230286598206, + "step": 37 + }, + { + "epoch": 0.4315117104329312, + "grad_norm": 2.1690878868103027, + "learning_rate": 1.9956981277233342e-05, + "loss": 1.0567874908447266, + "step": 38 + }, + { + "epoch": 0.44286728176011353, + "grad_norm": 2.005312442779541, + "learning_rate": 1.9943824622271934e-05, + "loss": 0.980779767036438, + "step": 39 + }, + { + "epoch": 0.45422285308729593, + "grad_norm": 2.0727038383483887, + "learning_rate": 1.9928920737019735e-05, + "loss": 0.8422978520393372, + "step": 40 + }, + { + "epoch": 0.46557842441447833, + "grad_norm": 2.4477312564849854, + "learning_rate": 1.9912272240239715e-05, + "loss": 1.239004373550415, + "step": 41 + }, + { + "epoch": 0.47693399574166073, + "grad_norm": 2.073049306869507, + "learning_rate": 1.9893882057240698e-05, + "loss": 1.0170605182647705, + "step": 42 + }, + { + "epoch": 0.48828956706884313, + "grad_norm": 1.9096711874008179, + "learning_rate": 1.9873753419363336e-05, + "loss": 0.7338559031486511, + "step": 43 + }, + { + "epoch": 0.49964513839602553, + "grad_norm": 1.514130711555481, + "learning_rate": 1.9851889863412347e-05, + "loss": 0.5721548795700073, + "step": 44 + }, + { + "epoch": 0.511000709723208, + "grad_norm": 1.8837509155273438, + "learning_rate": 1.9828295231035054e-05, + "loss": 0.7944353222846985, + "step": 45 + }, + { + "epoch": 0.5223562810503903, + "grad_norm": 1.6668717861175537, + "learning_rate": 1.9802973668046364e-05, + "loss": 0.703384518623352, + "step": 46 + }, + { + "epoch": 0.5337118523775728, + "grad_norm": 1.619913935661316, + "learning_rate": 1.9775929623700318e-05, + "loss": 0.7028679847717285, + "step": 47 + }, + { + "epoch": 0.5450674237047551, + "grad_norm": 1.797451138496399, + "learning_rate": 1.9747167849908305e-05, + "loss": 0.8451980948448181, + "step": 48 + }, + { + "epoch": 0.5564229950319376, + "grad_norm": 1.6591897010803223, + "learning_rate": 1.97166934004041e-05, + "loss": 0.6273907423019409, + "step": 49 + }, + { + "epoch": 0.5677785663591199, + "grad_norm": 1.6978833675384521, + "learning_rate": 1.968451162985589e-05, + "loss": 0.6259630918502808, + "step": 50 + }, + { + "epoch": 0.5677785663591199, + "eval_loss": 0.8047120571136475, + "eval_runtime": 27.6692, + "eval_samples_per_second": 5.168, + "eval_steps_per_second": 1.301, + "step": 50 + }, + { + "epoch": 0.5791341376863024, + "grad_norm": 2.0382018089294434, + "learning_rate": 1.9650628192925372e-05, + "loss": 0.8624173402786255, + "step": 51 + }, + { + "epoch": 0.5904897090134847, + "grad_norm": 2.2218334674835205, + "learning_rate": 1.9615049043274207e-05, + "loss": 1.0386168956756592, + "step": 52 + }, + { + "epoch": 0.6018452803406671, + "grad_norm": 1.7063782215118408, + "learning_rate": 1.957778043251788e-05, + "loss": 0.7194206118583679, + "step": 53 + }, + { + "epoch": 0.6132008516678495, + "grad_norm": 1.621178150177002, + "learning_rate": 1.953882890912723e-05, + "loss": 0.804326593875885, + "step": 54 + }, + { + "epoch": 0.6245564229950319, + "grad_norm": 1.5988848209381104, + "learning_rate": 1.949820131727783e-05, + "loss": 0.6942477822303772, + "step": 55 + }, + { + "epoch": 0.6359119943222143, + "grad_norm": 1.7638858556747437, + "learning_rate": 1.945590479564738e-05, + "loss": 0.688396692276001, + "step": 56 + }, + { + "epoch": 0.6472675656493967, + "grad_norm": 1.8592664003372192, + "learning_rate": 1.9411946776161388e-05, + "loss": 0.7517380118370056, + "step": 57 + }, + { + "epoch": 0.6586231369765791, + "grad_norm": 2.1580257415771484, + "learning_rate": 1.936633498268728e-05, + "loss": 1.1234959363937378, + "step": 58 + }, + { + "epoch": 0.6699787083037615, + "grad_norm": 2.0941476821899414, + "learning_rate": 1.931907742967727e-05, + "loss": 0.7969536185264587, + "step": 59 + }, + { + "epoch": 0.681334279630944, + "grad_norm": 1.8826228380203247, + "learning_rate": 1.9270182420760104e-05, + "loss": 0.7057633996009827, + "step": 60 + }, + { + "epoch": 0.6926898509581263, + "grad_norm": 1.8172886371612549, + "learning_rate": 1.921965854728207e-05, + "loss": 0.8432231545448303, + "step": 61 + }, + { + "epoch": 0.7040454222853088, + "grad_norm": 2.468703508377075, + "learning_rate": 1.916751468679737e-05, + "loss": 0.9716429710388184, + "step": 62 + }, + { + "epoch": 0.7154009936124911, + "grad_norm": 1.5727559328079224, + "learning_rate": 1.911376000150828e-05, + "loss": 0.6017226576805115, + "step": 63 + }, + { + "epoch": 0.7267565649396736, + "grad_norm": 1.9279130697250366, + "learning_rate": 1.9058403936655235e-05, + "loss": 0.945792019367218, + "step": 64 + }, + { + "epoch": 0.7381121362668559, + "grad_norm": 1.459025502204895, + "learning_rate": 1.9001456218857207e-05, + "loss": 0.601871907711029, + "step": 65 + }, + { + "epoch": 0.7494677075940384, + "grad_norm": 1.5340495109558105, + "learning_rate": 1.894292685440266e-05, + "loss": 0.6935733556747437, + "step": 66 + }, + { + "epoch": 0.7608232789212207, + "grad_norm": 1.6071991920471191, + "learning_rate": 1.888282612749132e-05, + "loss": 0.7782894968986511, + "step": 67 + }, + { + "epoch": 0.7721788502484032, + "grad_norm": 1.6310977935791016, + "learning_rate": 1.8821164598427148e-05, + "loss": 0.7309603095054626, + "step": 68 + }, + { + "epoch": 0.7835344215755855, + "grad_norm": 1.817807912826538, + "learning_rate": 1.8757953101762786e-05, + "loss": 0.790179967880249, + "step": 69 + }, + { + "epoch": 0.794889992902768, + "grad_norm": 1.3777976036071777, + "learning_rate": 1.869320274439583e-05, + "loss": 0.5821739435195923, + "step": 70 + }, + { + "epoch": 0.8062455642299503, + "grad_norm": 1.3368778228759766, + "learning_rate": 1.86269249036172e-05, + "loss": 0.5191158652305603, + "step": 71 + }, + { + "epoch": 0.8176011355571328, + "grad_norm": 1.7497543096542358, + "learning_rate": 1.8559131225112085e-05, + "loss": 0.7746997475624084, + "step": 72 + }, + { + "epoch": 0.8289567068843151, + "grad_norm": 1.6474581956863403, + "learning_rate": 1.8489833620913644e-05, + "loss": 0.5990749001502991, + "step": 73 + }, + { + "epoch": 0.8403122782114976, + "grad_norm": 1.425794005393982, + "learning_rate": 1.841904426730994e-05, + "loss": 0.6433415412902832, + "step": 74 + }, + { + "epoch": 0.8516678495386799, + "grad_norm": 1.8670772314071655, + "learning_rate": 1.8346775602704464e-05, + "loss": 0.840431809425354, + "step": 75 + }, + { + "epoch": 0.8630234208658624, + "grad_norm": 1.6109037399291992, + "learning_rate": 1.8273040325430575e-05, + "loss": 0.7096354961395264, + "step": 76 + }, + { + "epoch": 0.8743789921930447, + "grad_norm": 1.964331865310669, + "learning_rate": 1.8197851391520265e-05, + "loss": 0.7453011274337769, + "step": 77 + }, + { + "epoch": 0.8857345635202271, + "grad_norm": 1.7134088277816772, + "learning_rate": 1.8121222012427666e-05, + "loss": 0.7802327871322632, + "step": 78 + }, + { + "epoch": 0.8970901348474095, + "grad_norm": 1.964368224143982, + "learning_rate": 1.804316565270765e-05, + "loss": 0.8367159366607666, + "step": 79 + }, + { + "epoch": 0.9084457061745919, + "grad_norm": 1.9737969636917114, + "learning_rate": 1.796369602764999e-05, + "loss": 0.8365910053253174, + "step": 80 + }, + { + "epoch": 0.9198012775017743, + "grad_norm": 1.963098168373108, + "learning_rate": 1.788282710086942e-05, + "loss": 0.9032438397407532, + "step": 81 + }, + { + "epoch": 0.9311568488289567, + "grad_norm": 1.771173119544983, + "learning_rate": 1.7800573081852124e-05, + "loss": 0.7875261902809143, + "step": 82 + }, + { + "epoch": 0.9425124201561391, + "grad_norm": 1.9102442264556885, + "learning_rate": 1.771694842345894e-05, + "loss": 0.76175856590271, + "step": 83 + }, + { + "epoch": 0.9538679914833215, + "grad_norm": 2.03609299659729, + "learning_rate": 1.7631967819385883e-05, + "loss": 0.8382056951522827, + "step": 84 + }, + { + "epoch": 0.9652235628105039, + "grad_norm": 1.864434003829956, + "learning_rate": 1.7545646201582304e-05, + "loss": 0.785956084728241, + "step": 85 + }, + { + "epoch": 0.9765791341376863, + "grad_norm": 2.0304460525512695, + "learning_rate": 1.7457998737627183e-05, + "loss": 0.9167435169219971, + "step": 86 + }, + { + "epoch": 0.9879347054648687, + "grad_norm": 1.691648006439209, + "learning_rate": 1.7369040828064046e-05, + "loss": 0.7328086495399475, + "step": 87 + }, + { + "epoch": 0.9992902767920511, + "grad_norm": 2.0275533199310303, + "learning_rate": 1.7278788103694944e-05, + "loss": 0.9172717928886414, + "step": 88 + }, + { + "epoch": 1.0, + "grad_norm": 2.6658289432525635, + "learning_rate": 1.7187256422833928e-05, + "loss": 0.11026950925588608, + "step": 89 + }, + { + "epoch": 1.0113555713271825, + "grad_norm": 1.5766886472702026, + "learning_rate": 1.7094461868520625e-05, + "loss": 0.7454363107681274, + "step": 90 + }, + { + "epoch": 1.0227111426543647, + "grad_norm": 1.934645652770996, + "learning_rate": 1.7000420745694256e-05, + "loss": 0.9695757627487183, + "step": 91 + }, + { + "epoch": 1.0340667139815471, + "grad_norm": 1.7055552005767822, + "learning_rate": 1.6905149578328705e-05, + "loss": 0.7050992846488953, + "step": 92 + }, + { + "epoch": 1.0454222853087296, + "grad_norm": 1.5468671321868896, + "learning_rate": 1.6808665106529096e-05, + "loss": 0.6628597974777222, + "step": 93 + }, + { + "epoch": 1.056777856635912, + "grad_norm": 1.80152428150177, + "learning_rate": 1.671098428359037e-05, + "loss": 0.826714277267456, + "step": 94 + }, + { + "epoch": 1.0681334279630943, + "grad_norm": 1.9063488245010376, + "learning_rate": 1.661212427301844e-05, + "loss": 0.9870727062225342, + "step": 95 + }, + { + "epoch": 1.0794889992902768, + "grad_norm": 1.7208439111709595, + "learning_rate": 1.6512102445514376e-05, + "loss": 0.754402220249176, + "step": 96 + }, + { + "epoch": 1.0908445706174592, + "grad_norm": 1.688193440437317, + "learning_rate": 1.64109363759222e-05, + "loss": 0.6640122532844543, + "step": 97 + }, + { + "epoch": 1.1022001419446417, + "grad_norm": 1.8576338291168213, + "learning_rate": 1.630864384014083e-05, + "loss": 0.8575501441955566, + "step": 98 + }, + { + "epoch": 1.113555713271824, + "grad_norm": 1.7428895235061646, + "learning_rate": 1.620524281200062e-05, + "loss": 0.5853228569030762, + "step": 99 + }, + { + "epoch": 1.1249112845990064, + "grad_norm": 1.5940732955932617, + "learning_rate": 1.6100751460105244e-05, + "loss": 0.6318332552909851, + "step": 100 + }, + { + "epoch": 1.1249112845990064, + "eval_loss": 0.7593621015548706, + "eval_runtime": 27.6141, + "eval_samples_per_second": 5.179, + "eval_steps_per_second": 1.304, + "step": 100 + }, + { + "epoch": 1.1362668559261888, + "grad_norm": 1.9089066982269287, + "learning_rate": 1.599518814463925e-05, + "loss": 0.9151409268379211, + "step": 101 + }, + { + "epoch": 1.1476224272533713, + "grad_norm": 1.905922770500183, + "learning_rate": 1.5888571414141997e-05, + "loss": 0.8580778241157532, + "step": 102 + }, + { + "epoch": 1.1589779985805535, + "grad_norm": 1.964565634727478, + "learning_rate": 1.5780920002248484e-05, + "loss": 0.8872693777084351, + "step": 103 + }, + { + "epoch": 1.170333569907736, + "grad_norm": 2.039443254470825, + "learning_rate": 1.5672252824397683e-05, + "loss": 0.7390342950820923, + "step": 104 + }, + { + "epoch": 1.1816891412349184, + "grad_norm": 1.685909390449524, + "learning_rate": 1.556258897450887e-05, + "loss": 0.6857212781906128, + "step": 105 + }, + { + "epoch": 1.1930447125621009, + "grad_norm": 1.9805678129196167, + "learning_rate": 1.5451947721626676e-05, + "loss": 0.685741662979126, + "step": 106 + }, + { + "epoch": 1.204400283889283, + "grad_norm": 1.6393771171569824, + "learning_rate": 1.534034850653528e-05, + "loss": 0.6081647276878357, + "step": 107 + }, + { + "epoch": 1.2157558552164656, + "grad_norm": 1.371028184890747, + "learning_rate": 1.5227810938342493e-05, + "loss": 0.44291260838508606, + "step": 108 + }, + { + "epoch": 1.227111426543648, + "grad_norm": 1.8976811170578003, + "learning_rate": 1.5114354791034225e-05, + "loss": 0.82900071144104, + "step": 109 + }, + { + "epoch": 1.2384669978708303, + "grad_norm": 1.4269609451293945, + "learning_rate": 1.5000000000000002e-05, + "loss": 0.44125357270240784, + "step": 110 + }, + { + "epoch": 1.2498225691980127, + "grad_norm": 1.9113351106643677, + "learning_rate": 1.4884766658530126e-05, + "loss": 0.7995925545692444, + "step": 111 + }, + { + "epoch": 1.2611781405251952, + "grad_norm": 1.9662408828735352, + "learning_rate": 1.4768675014285063e-05, + "loss": 0.8006548881530762, + "step": 112 + }, + { + "epoch": 1.2725337118523776, + "grad_norm": 2.118868350982666, + "learning_rate": 1.465174546573774e-05, + "loss": 1.086897611618042, + "step": 113 + }, + { + "epoch": 1.28388928317956, + "grad_norm": 1.7964507341384888, + "learning_rate": 1.4533998558589319e-05, + "loss": 0.7104189395904541, + "step": 114 + }, + { + "epoch": 1.2952448545067423, + "grad_norm": 1.659604549407959, + "learning_rate": 1.4415454982159121e-05, + "loss": 0.7449539303779602, + "step": 115 + }, + { + "epoch": 1.3066004258339248, + "grad_norm": 1.7559200525283813, + "learning_rate": 1.429613556574928e-05, + "loss": 0.760001540184021, + "step": 116 + }, + { + "epoch": 1.3179559971611072, + "grad_norm": 1.6712889671325684, + "learning_rate": 1.4176061274984858e-05, + "loss": 0.5844006538391113, + "step": 117 + }, + { + "epoch": 1.3293115684882895, + "grad_norm": 1.8857864141464233, + "learning_rate": 1.405525320812994e-05, + "loss": 0.799013614654541, + "step": 118 + }, + { + "epoch": 1.340667139815472, + "grad_norm": 1.9506901502609253, + "learning_rate": 1.3933732592380485e-05, + "loss": 0.8254635334014893, + "step": 119 + }, + { + "epoch": 1.3520227111426544, + "grad_norm": 1.556196689605713, + "learning_rate": 1.3811520780134471e-05, + "loss": 0.6055241227149963, + "step": 120 + }, + { + "epoch": 1.3633782824698368, + "grad_norm": 1.5888230800628662, + "learning_rate": 1.3688639245240078e-05, + "loss": 0.641814112663269, + "step": 121 + }, + { + "epoch": 1.3747338537970193, + "grad_norm": 1.740252137184143, + "learning_rate": 1.3565109579222511e-05, + "loss": 0.746912956237793, + "step": 122 + }, + { + "epoch": 1.3860894251242015, + "grad_norm": 2.0926947593688965, + "learning_rate": 1.3440953487490145e-05, + "loss": 0.8887531161308289, + "step": 123 + }, + { + "epoch": 1.397444996451384, + "grad_norm": 1.7420989274978638, + "learning_rate": 1.331619278552068e-05, + "loss": 0.6902853846549988, + "step": 124 + }, + { + "epoch": 1.4088005677785664, + "grad_norm": 1.8094593286514282, + "learning_rate": 1.3190849395027926e-05, + "loss": 0.7649242877960205, + "step": 125 + }, + { + "epoch": 1.4201561391057487, + "grad_norm": 1.4437446594238281, + "learning_rate": 1.306494534010995e-05, + "loss": 0.5692201256752014, + "step": 126 + }, + { + "epoch": 1.4315117104329311, + "grad_norm": 1.3073190450668335, + "learning_rate": 1.2938502743379212e-05, + "loss": 0.4875521957874298, + "step": 127 + }, + { + "epoch": 1.4428672817601136, + "grad_norm": 1.6500370502471924, + "learning_rate": 1.2811543822075396e-05, + "loss": 0.6286922097206116, + "step": 128 + }, + { + "epoch": 1.454222853087296, + "grad_norm": 1.4302570819854736, + "learning_rate": 1.2684090884161636e-05, + "loss": 0.48701754212379456, + "step": 129 + }, + { + "epoch": 1.4655784244144783, + "grad_norm": 1.7269134521484375, + "learning_rate": 1.2556166324404747e-05, + "loss": 0.6981077790260315, + "step": 130 + }, + { + "epoch": 1.4769339957416607, + "grad_norm": 1.5457602739334106, + "learning_rate": 1.242779262044028e-05, + "loss": 0.48003125190734863, + "step": 131 + }, + { + "epoch": 1.4882895670688432, + "grad_norm": 1.8600218296051025, + "learning_rate": 1.2298992328822937e-05, + "loss": 0.7610920667648315, + "step": 132 + }, + { + "epoch": 1.4996451383960254, + "grad_norm": 1.3750872611999512, + "learning_rate": 1.2169788081063181e-05, + "loss": 0.4584071934223175, + "step": 133 + }, + { + "epoch": 1.5110007097232079, + "grad_norm": 1.5970045328140259, + "learning_rate": 1.2040202579650649e-05, + "loss": 0.6092408895492554, + "step": 134 + }, + { + "epoch": 1.5223562810503903, + "grad_norm": 1.8978732824325562, + "learning_rate": 1.1910258594065079e-05, + "loss": 0.8766818642616272, + "step": 135 + }, + { + "epoch": 1.5337118523775728, + "grad_norm": 1.7095293998718262, + "learning_rate": 1.1779978956775507e-05, + "loss": 0.629298985004425, + "step": 136 + }, + { + "epoch": 1.5450674237047552, + "grad_norm": 1.8608379364013672, + "learning_rate": 1.1649386559228342e-05, + "loss": 0.6764330267906189, + "step": 137 + }, + { + "epoch": 1.5564229950319377, + "grad_norm": 1.760704517364502, + "learning_rate": 1.1518504347825146e-05, + "loss": 0.7110517621040344, + "step": 138 + }, + { + "epoch": 1.56777856635912, + "grad_norm": 1.937847375869751, + "learning_rate": 1.1387355319890685e-05, + "loss": 0.7304476499557495, + "step": 139 + }, + { + "epoch": 1.5791341376863024, + "grad_norm": 1.5163425207138062, + "learning_rate": 1.1255962519632082e-05, + "loss": 0.6055701971054077, + "step": 140 + }, + { + "epoch": 1.5904897090134846, + "grad_norm": 1.6075628995895386, + "learning_rate": 1.1124349034089724e-05, + "loss": 0.5708965063095093, + "step": 141 + }, + { + "epoch": 1.601845280340667, + "grad_norm": 1.7377432584762573, + "learning_rate": 1.0992537989080618e-05, + "loss": 0.672942042350769, + "step": 142 + }, + { + "epoch": 1.6132008516678495, + "grad_norm": 2.052077293395996, + "learning_rate": 1.086055254513497e-05, + "loss": 0.7728007435798645, + "step": 143 + }, + { + "epoch": 1.624556422995032, + "grad_norm": 1.5325052738189697, + "learning_rate": 1.0728415893426636e-05, + "loss": 0.5753393769264221, + "step": 144 + }, + { + "epoch": 1.6359119943222145, + "grad_norm": 2.0628015995025635, + "learning_rate": 1.05961512516982e-05, + "loss": 0.7965511083602905, + "step": 145 + }, + { + "epoch": 1.6472675656493967, + "grad_norm": 1.5534552335739136, + "learning_rate": 1.0463781860181385e-05, + "loss": 0.4485720694065094, + "step": 146 + }, + { + "epoch": 1.6586231369765791, + "grad_norm": 2.0554072856903076, + "learning_rate": 1.033133097751351e-05, + "loss": 0.87266606092453, + "step": 147 + }, + { + "epoch": 1.6699787083037614, + "grad_norm": 1.702256441116333, + "learning_rate": 1.0198821876650702e-05, + "loss": 0.5705003142356873, + "step": 148 + }, + { + "epoch": 1.6813342796309438, + "grad_norm": 1.5394976139068604, + "learning_rate": 1.0066277840778626e-05, + "loss": 0.4785357117652893, + "step": 149 + }, + { + "epoch": 1.6926898509581263, + "grad_norm": 1.7153429985046387, + "learning_rate": 9.933722159221375e-06, + "loss": 0.7122579216957092, + "step": 150 + }, + { + "epoch": 1.6926898509581263, + "eval_loss": 0.7425792217254639, + "eval_runtime": 27.5719, + "eval_samples_per_second": 5.186, + "eval_steps_per_second": 1.306, + "step": 150 + }, + { + "epoch": 1.7040454222853088, + "grad_norm": 1.7121310234069824, + "learning_rate": 9.801178123349298e-06, + "loss": 0.6155564188957214, + "step": 151 + }, + { + "epoch": 1.7154009936124912, + "grad_norm": 1.8881406784057617, + "learning_rate": 9.668669022486495e-06, + "loss": 0.7582035064697266, + "step": 152 + }, + { + "epoch": 1.7267565649396737, + "grad_norm": 1.9322803020477295, + "learning_rate": 9.536218139818615e-06, + "loss": 0.8061675429344177, + "step": 153 + }, + { + "epoch": 1.738112136266856, + "grad_norm": 1.5622960329055786, + "learning_rate": 9.403848748301802e-06, + "loss": 0.5297854542732239, + "step": 154 + }, + { + "epoch": 1.7494677075940384, + "grad_norm": 1.8679121732711792, + "learning_rate": 9.271584106573364e-06, + "loss": 0.6446192264556885, + "step": 155 + }, + { + "epoch": 1.7608232789212206, + "grad_norm": 1.65900456905365, + "learning_rate": 9.139447454865034e-06, + "loss": 0.6376427412033081, + "step": 156 + }, + { + "epoch": 1.772178850248403, + "grad_norm": 2.008634328842163, + "learning_rate": 9.007462010919387e-06, + "loss": 0.8298434615135193, + "step": 157 + }, + { + "epoch": 1.7835344215755855, + "grad_norm": 1.817112684249878, + "learning_rate": 8.87565096591028e-06, + "loss": 0.6945917010307312, + "step": 158 + }, + { + "epoch": 1.794889992902768, + "grad_norm": 2.0954995155334473, + "learning_rate": 8.744037480367922e-06, + "loss": 0.922713577747345, + "step": 159 + }, + { + "epoch": 1.8062455642299504, + "grad_norm": 1.8708635568618774, + "learning_rate": 8.61264468010932e-06, + "loss": 0.7695997953414917, + "step": 160 + }, + { + "epoch": 1.8176011355571329, + "grad_norm": 2.0717625617980957, + "learning_rate": 8.481495652174859e-06, + "loss": 0.9634745121002197, + "step": 161 + }, + { + "epoch": 1.828956706884315, + "grad_norm": 1.4181921482086182, + "learning_rate": 8.350613440771661e-06, + "loss": 0.41311249136924744, + "step": 162 + }, + { + "epoch": 1.8403122782114976, + "grad_norm": 1.5503257513046265, + "learning_rate": 8.2200210432245e-06, + "loss": 0.6834385395050049, + "step": 163 + }, + { + "epoch": 1.8516678495386798, + "grad_norm": 1.767377257347107, + "learning_rate": 8.089741405934923e-06, + "loss": 0.7185457944869995, + "step": 164 + }, + { + "epoch": 1.8630234208658623, + "grad_norm": 1.8889371156692505, + "learning_rate": 7.959797420349356e-06, + "loss": 0.9002049565315247, + "step": 165 + }, + { + "epoch": 1.8743789921930447, + "grad_norm": 2.01930832862854, + "learning_rate": 7.83021191893682e-06, + "loss": 0.6417741775512695, + "step": 166 + }, + { + "epoch": 1.8857345635202272, + "grad_norm": 1.8323336839675903, + "learning_rate": 7.701007671177066e-06, + "loss": 0.8583472371101379, + "step": 167 + }, + { + "epoch": 1.8970901348474096, + "grad_norm": 1.3641561269760132, + "learning_rate": 7.572207379559722e-06, + "loss": 0.5814041495323181, + "step": 168 + }, + { + "epoch": 1.9084457061745919, + "grad_norm": 1.6025844812393188, + "learning_rate": 7.443833675595254e-06, + "loss": 0.5391830205917358, + "step": 169 + }, + { + "epoch": 1.9198012775017743, + "grad_norm": 1.523732304573059, + "learning_rate": 7.315909115838367e-06, + "loss": 0.6024059057235718, + "step": 170 + }, + { + "epoch": 1.9311568488289566, + "grad_norm": 1.6438531875610352, + "learning_rate": 7.1884561779246055e-06, + "loss": 0.5504345297813416, + "step": 171 + }, + { + "epoch": 1.942512420156139, + "grad_norm": 1.884101152420044, + "learning_rate": 7.061497256620793e-06, + "loss": 0.8089767694473267, + "step": 172 + }, + { + "epoch": 1.9538679914833215, + "grad_norm": 1.705494999885559, + "learning_rate": 6.935054659890053e-06, + "loss": 0.6709920763969421, + "step": 173 + }, + { + "epoch": 1.965223562810504, + "grad_norm": 1.8286340236663818, + "learning_rate": 6.809150604972079e-06, + "loss": 0.9714908003807068, + "step": 174 + }, + { + "epoch": 1.9765791341376864, + "grad_norm": 1.6881917715072632, + "learning_rate": 6.683807214479323e-06, + "loss": 0.6124534606933594, + "step": 175 + }, + { + "epoch": 1.9879347054648688, + "grad_norm": 2.147641181945801, + "learning_rate": 6.559046512509859e-06, + "loss": 0.9034600257873535, + "step": 176 + }, + { + "epoch": 1.999290276792051, + "grad_norm": 1.7281945943832397, + "learning_rate": 6.434890420777491e-06, + "loss": 0.7872213125228882, + "step": 177 + }, + { + "epoch": 2.0, + "grad_norm": 4.5360822677612305, + "learning_rate": 6.311360754759923e-06, + "loss": 0.2226628065109253, + "step": 178 + }, + { + "epoch": 2.0113555713271825, + "grad_norm": 1.6504145860671997, + "learning_rate": 6.188479219865529e-06, + "loss": 0.6821281313896179, + "step": 179 + }, + { + "epoch": 2.022711142654365, + "grad_norm": 1.4790889024734497, + "learning_rate": 6.06626740761952e-06, + "loss": 0.5527986288070679, + "step": 180 + }, + { + "epoch": 2.0340667139815474, + "grad_norm": 1.8767532110214233, + "learning_rate": 5.944746791870062e-06, + "loss": 0.7406971454620361, + "step": 181 + }, + { + "epoch": 2.0454222853087294, + "grad_norm": 1.6067395210266113, + "learning_rate": 5.823938725015148e-06, + "loss": 0.5897183418273926, + "step": 182 + }, + { + "epoch": 2.056777856635912, + "grad_norm": 1.6543129682540894, + "learning_rate": 5.703864434250721e-06, + "loss": 0.5693725943565369, + "step": 183 + }, + { + "epoch": 2.0681334279630943, + "grad_norm": 1.763333797454834, + "learning_rate": 5.584545017840886e-06, + "loss": 0.6083229184150696, + "step": 184 + }, + { + "epoch": 2.0794889992902768, + "grad_norm": 1.2259186506271362, + "learning_rate": 5.4660014414106825e-06, + "loss": 0.3644542098045349, + "step": 185 + }, + { + "epoch": 2.090844570617459, + "grad_norm": 1.6024012565612793, + "learning_rate": 5.348254534262262e-06, + "loss": 0.607455313205719, + "step": 186 + }, + { + "epoch": 2.1022001419446417, + "grad_norm": 1.5699106454849243, + "learning_rate": 5.231324985714942e-06, + "loss": 0.4221411645412445, + "step": 187 + }, + { + "epoch": 2.113555713271824, + "grad_norm": 1.439180850982666, + "learning_rate": 5.1152333414698774e-06, + "loss": 0.4416266679763794, + "step": 188 + }, + { + "epoch": 2.1249112845990066, + "grad_norm": 1.8221555948257446, + "learning_rate": 5.000000000000003e-06, + "loss": 0.5936031341552734, + "step": 189 + }, + { + "epoch": 2.1362668559261886, + "grad_norm": 1.888954520225525, + "learning_rate": 4.885645208965779e-06, + "loss": 0.7918609976768494, + "step": 190 + }, + { + "epoch": 2.147622427253371, + "grad_norm": 1.6411021947860718, + "learning_rate": 4.772189061657511e-06, + "loss": 0.5752283334732056, + "step": 191 + }, + { + "epoch": 2.1589779985805535, + "grad_norm": 1.6003109216690063, + "learning_rate": 4.659651493464721e-06, + "loss": 0.6083833575248718, + "step": 192 + }, + { + "epoch": 2.170333569907736, + "grad_norm": 1.657971739768982, + "learning_rate": 4.548052278373327e-06, + "loss": 0.6448208093643188, + "step": 193 + }, + { + "epoch": 2.1816891412349184, + "grad_norm": 1.9829140901565552, + "learning_rate": 4.437411025491131e-06, + "loss": 0.6960307359695435, + "step": 194 + }, + { + "epoch": 2.193044712562101, + "grad_norm": 1.8921784162521362, + "learning_rate": 4.327747175602321e-06, + "loss": 0.731373131275177, + "step": 195 + }, + { + "epoch": 2.2044002838892833, + "grad_norm": 1.7255470752716064, + "learning_rate": 4.219079997751515e-06, + "loss": 0.5975107550621033, + "step": 196 + }, + { + "epoch": 2.215755855216466, + "grad_norm": 1.5999464988708496, + "learning_rate": 4.111428585858005e-06, + "loss": 0.5760666131973267, + "step": 197 + }, + { + "epoch": 2.227111426543648, + "grad_norm": 1.240045428276062, + "learning_rate": 4.0048118553607485e-06, + "loss": 0.39564967155456543, + "step": 198 + }, + { + "epoch": 2.2384669978708303, + "grad_norm": 1.9376376867294312, + "learning_rate": 3.899248539894756e-06, + "loss": 0.6683239936828613, + "step": 199 + }, + { + "epoch": 2.2498225691980127, + "grad_norm": 1.8060548305511475, + "learning_rate": 3.794757187999386e-06, + "loss": 0.7160100936889648, + "step": 200 + }, + { + "epoch": 2.2498225691980127, + "eval_loss": 0.7327238321304321, + "eval_runtime": 27.6122, + "eval_samples_per_second": 5.179, + "eval_steps_per_second": 1.304, + "step": 200 + }, + { + "epoch": 2.261178140525195, + "grad_norm": 1.5847166776657104, + "learning_rate": 3.6913561598591775e-06, + "loss": 0.5229685306549072, + "step": 201 + }, + { + "epoch": 2.2725337118523776, + "grad_norm": 1.6213287115097046, + "learning_rate": 3.589063624077802e-06, + "loss": 0.5970481634140015, + "step": 202 + }, + { + "epoch": 2.28388928317956, + "grad_norm": 1.818298578262329, + "learning_rate": 3.4878975544856285e-06, + "loss": 0.7519898414611816, + "step": 203 + }, + { + "epoch": 2.2952448545067425, + "grad_norm": 1.6277714967727661, + "learning_rate": 3.387875726981563e-06, + "loss": 0.5007752180099487, + "step": 204 + }, + { + "epoch": 2.306600425833925, + "grad_norm": 1.6472209692001343, + "learning_rate": 3.2890157164096315e-06, + "loss": 0.6382125616073608, + "step": 205 + }, + { + "epoch": 2.317955997161107, + "grad_norm": 1.821578025817871, + "learning_rate": 3.1913348934709076e-06, + "loss": 0.7000253200531006, + "step": 206 + }, + { + "epoch": 2.3293115684882895, + "grad_norm": 1.8459348678588867, + "learning_rate": 3.094850421671295e-06, + "loss": 0.5165408253669739, + "step": 207 + }, + { + "epoch": 2.340667139815472, + "grad_norm": 2.36333966255188, + "learning_rate": 2.999579254305748e-06, + "loss": 0.8205851912498474, + "step": 208 + }, + { + "epoch": 2.3520227111426544, + "grad_norm": 1.6719001531600952, + "learning_rate": 2.905538131479376e-06, + "loss": 0.5079140663146973, + "step": 209 + }, + { + "epoch": 2.363378282469837, + "grad_norm": 1.5651746988296509, + "learning_rate": 2.812743577166075e-06, + "loss": 0.4731737971305847, + "step": 210 + }, + { + "epoch": 2.3747338537970193, + "grad_norm": 2.142404317855835, + "learning_rate": 2.721211896305059e-06, + "loss": 0.842810869216919, + "step": 211 + }, + { + "epoch": 2.3860894251242017, + "grad_norm": 1.796759843826294, + "learning_rate": 2.6309591719359563e-06, + "loss": 0.7187446355819702, + "step": 212 + }, + { + "epoch": 2.3974449964513838, + "grad_norm": 1.492469310760498, + "learning_rate": 2.542001262372821e-06, + "loss": 0.5724196434020996, + "step": 213 + }, + { + "epoch": 2.408800567778566, + "grad_norm": 1.905091643333435, + "learning_rate": 2.454353798417698e-06, + "loss": 0.8085651397705078, + "step": 214 + }, + { + "epoch": 2.4201561391057487, + "grad_norm": 1.7704670429229736, + "learning_rate": 2.3680321806141182e-06, + "loss": 0.6132161021232605, + "step": 215 + }, + { + "epoch": 2.431511710432931, + "grad_norm": 1.8361713886260986, + "learning_rate": 2.283051576541062e-06, + "loss": 0.7117694616317749, + "step": 216 + }, + { + "epoch": 2.4428672817601136, + "grad_norm": 1.6990126371383667, + "learning_rate": 2.19942691814788e-06, + "loss": 0.6097843050956726, + "step": 217 + }, + { + "epoch": 2.454222853087296, + "grad_norm": 1.7435643672943115, + "learning_rate": 2.1171728991305797e-06, + "loss": 0.5934941172599792, + "step": 218 + }, + { + "epoch": 2.4655784244144785, + "grad_norm": 1.9126805067062378, + "learning_rate": 2.0363039723500155e-06, + "loss": 0.704249918460846, + "step": 219 + }, + { + "epoch": 2.4769339957416605, + "grad_norm": 1.7842662334442139, + "learning_rate": 1.9568343472923524e-06, + "loss": 0.6474202871322632, + "step": 220 + }, + { + "epoch": 2.488289567068843, + "grad_norm": 1.7116307020187378, + "learning_rate": 1.8787779875723389e-06, + "loss": 0.5358954668045044, + "step": 221 + }, + { + "epoch": 2.4996451383960254, + "grad_norm": 1.9682855606079102, + "learning_rate": 1.8021486084797368e-06, + "loss": 0.7930698990821838, + "step": 222 + }, + { + "epoch": 2.511000709723208, + "grad_norm": 1.4548183679580688, + "learning_rate": 1.7269596745694295e-06, + "loss": 0.4583667814731598, + "step": 223 + }, + { + "epoch": 2.5223562810503903, + "grad_norm": 2.000056505203247, + "learning_rate": 1.6532243972955397e-06, + "loss": 0.8167339563369751, + "step": 224 + }, + { + "epoch": 2.533711852377573, + "grad_norm": 2.187757730484009, + "learning_rate": 1.580955732690065e-06, + "loss": 0.8731608390808105, + "step": 225 + }, + { + "epoch": 2.5450674237047552, + "grad_norm": 2.552974224090576, + "learning_rate": 1.5101663790863597e-06, + "loss": 0.9876814484596252, + "step": 226 + }, + { + "epoch": 2.5564229950319377, + "grad_norm": 2.047971725463867, + "learning_rate": 1.4408687748879157e-06, + "loss": 0.7313542366027832, + "step": 227 + }, + { + "epoch": 2.56777856635912, + "grad_norm": 2.023643732070923, + "learning_rate": 1.3730750963828033e-06, + "loss": 0.7768765091896057, + "step": 228 + }, + { + "epoch": 2.579134137686302, + "grad_norm": 1.7678884267807007, + "learning_rate": 1.3067972556041753e-06, + "loss": 0.5483944416046143, + "step": 229 + }, + { + "epoch": 2.5904897090134846, + "grad_norm": 1.57295560836792, + "learning_rate": 1.2420468982372158e-06, + "loss": 0.5286450386047363, + "step": 230 + }, + { + "epoch": 2.601845280340667, + "grad_norm": 2.199338912963867, + "learning_rate": 1.1788354015728543e-06, + "loss": 0.7952181100845337, + "step": 231 + }, + { + "epoch": 2.6132008516678495, + "grad_norm": 2.037968635559082, + "learning_rate": 1.1171738725086833e-06, + "loss": 0.8746033906936646, + "step": 232 + }, + { + "epoch": 2.624556422995032, + "grad_norm": 1.6114685535430908, + "learning_rate": 1.0570731455973415e-06, + "loss": 0.5900834202766418, + "step": 233 + }, + { + "epoch": 2.6359119943222145, + "grad_norm": 1.8988250494003296, + "learning_rate": 9.985437811427934e-07, + "loss": 0.7454937696456909, + "step": 234 + }, + { + "epoch": 2.647267565649397, + "grad_norm": 1.6500483751296997, + "learning_rate": 9.415960633447674e-07, + "loss": 0.5034666657447815, + "step": 235 + }, + { + "epoch": 2.658623136976579, + "grad_norm": 1.9361827373504639, + "learning_rate": 8.862399984917214e-07, + "loss": 0.651042103767395, + "step": 236 + }, + { + "epoch": 2.6699787083037614, + "grad_norm": 1.9240092039108276, + "learning_rate": 8.3248531320263e-07, + "loss": 0.8603060245513916, + "step": 237 + }, + { + "epoch": 2.681334279630944, + "grad_norm": 1.9688630104064941, + "learning_rate": 7.803414527179343e-07, + "loss": 0.8090041875839233, + "step": 238 + }, + { + "epoch": 2.6926898509581263, + "grad_norm": 1.7253657579421997, + "learning_rate": 7.298175792398976e-07, + "loss": 0.5564637184143066, + "step": 239 + }, + { + "epoch": 2.7040454222853088, + "grad_norm": 1.871412754058838, + "learning_rate": 6.809225703227352e-07, + "loss": 0.7740951776504517, + "step": 240 + }, + { + "epoch": 2.715400993612491, + "grad_norm": 1.8274517059326172, + "learning_rate": 6.336650173127224e-07, + "loss": 0.6926577091217041, + "step": 241 + }, + { + "epoch": 2.7267565649396737, + "grad_norm": 2.128312349319458, + "learning_rate": 5.880532238386161e-07, + "loss": 0.8623908758163452, + "step": 242 + }, + { + "epoch": 2.7381121362668557, + "grad_norm": 1.9063048362731934, + "learning_rate": 5.440952043526215e-07, + "loss": 0.642934262752533, + "step": 243 + }, + { + "epoch": 2.7494677075940386, + "grad_norm": 1.9280798435211182, + "learning_rate": 5.017986827221733e-07, + "loss": 0.5877834558486938, + "step": 244 + }, + { + "epoch": 2.7608232789212206, + "grad_norm": 1.7953683137893677, + "learning_rate": 4.61171090872774e-07, + "loss": 0.6689532995223999, + "step": 245 + }, + { + "epoch": 2.772178850248403, + "grad_norm": 2.076279640197754, + "learning_rate": 4.222195674821239e-07, + "loss": 0.8235094547271729, + "step": 246 + }, + { + "epoch": 2.7835344215755855, + "grad_norm": 2.2308530807495117, + "learning_rate": 3.8495095672579584e-07, + "loss": 0.8819231986999512, + "step": 247 + }, + { + "epoch": 2.794889992902768, + "grad_norm": 1.5626147985458374, + "learning_rate": 3.493718070746299e-07, + "loss": 0.45997241139411926, + "step": 248 + }, + { + "epoch": 2.8062455642299504, + "grad_norm": 1.80852472782135, + "learning_rate": 3.154883701441136e-07, + "loss": 0.5900078415870667, + "step": 249 + }, + { + "epoch": 2.817601135557133, + "grad_norm": 1.591232180595398, + "learning_rate": 2.8330659959589944e-07, + "loss": 0.4933575987815857, + "step": 250 + }, + { + "epoch": 2.817601135557133, + "eval_loss": 0.7293056845664978, + "eval_runtime": 27.6187, + "eval_samples_per_second": 5.178, + "eval_steps_per_second": 1.303, + "step": 250 + }, + { + "epoch": 2.8289567068843153, + "grad_norm": 1.8240505456924438, + "learning_rate": 2.528321500916986e-07, + "loss": 0.6979119777679443, + "step": 251 + }, + { + "epoch": 2.8403122782114973, + "grad_norm": 1.9255794286727905, + "learning_rate": 2.2407037629968431e-07, + "loss": 0.7218936681747437, + "step": 252 + }, + { + "epoch": 2.85166784953868, + "grad_norm": 1.9031062126159668, + "learning_rate": 1.9702633195363918e-07, + "loss": 0.7347554564476013, + "step": 253 + }, + { + "epoch": 2.8630234208658623, + "grad_norm": 2.0092129707336426, + "learning_rate": 1.717047689649487e-07, + "loss": 0.8473051190376282, + "step": 254 + }, + { + "epoch": 2.8743789921930447, + "grad_norm": 2.1279916763305664, + "learning_rate": 1.4811013658765471e-07, + "loss": 0.8153268098831177, + "step": 255 + }, + { + "epoch": 2.885734563520227, + "grad_norm": 1.5587924718856812, + "learning_rate": 1.262465806366664e-07, + "loss": 0.42626267671585083, + "step": 256 + }, + { + "epoch": 2.8970901348474096, + "grad_norm": 1.851258635520935, + "learning_rate": 1.0611794275930398e-07, + "loss": 0.8002920150756836, + "step": 257 + }, + { + "epoch": 2.908445706174592, + "grad_norm": 1.7526359558105469, + "learning_rate": 8.772775976028547e-08, + "loss": 0.8035959005355835, + "step": 258 + }, + { + "epoch": 2.919801277501774, + "grad_norm": 1.8454657793045044, + "learning_rate": 7.10792629802659e-08, + "loss": 0.6260057687759399, + "step": 259 + }, + { + "epoch": 2.9311568488289566, + "grad_norm": 1.751435399055481, + "learning_rate": 5.617537772806603e-08, + "loss": 0.6086565256118774, + "step": 260 + }, + { + "epoch": 2.942512420156139, + "grad_norm": 1.614911437034607, + "learning_rate": 4.3018722766661194e-08, + "loss": 0.5733129382133484, + "step": 261 + }, + { + "epoch": 2.9538679914833215, + "grad_norm": 1.6565971374511719, + "learning_rate": 3.161160985304168e-08, + "loss": 0.45287761092185974, + "step": 262 + }, + { + "epoch": 2.965223562810504, + "grad_norm": 1.740827202796936, + "learning_rate": 2.1956043332010957e-08, + "loss": 0.5719068646430969, + "step": 263 + }, + { + "epoch": 2.9765791341376864, + "grad_norm": 1.4406249523162842, + "learning_rate": 1.405371978400516e-08, + "loss": 0.5211077332496643, + "step": 264 + }, + { + "epoch": 2.987934705464869, + "grad_norm": 1.6702423095703125, + "learning_rate": 7.906027726981568e-09, + "loss": 0.5729439854621887, + "step": 265 + }, + { + "epoch": 2.999290276792051, + "grad_norm": 1.812246561050415, + "learning_rate": 3.514047372448204e-09, + "loss": 0.6405225396156311, + "step": 266 + }, + { + "epoch": 3.0, + "grad_norm": 8.397170066833496, + "learning_rate": 8.785504356556562e-10, + "loss": 1.24993097782135, + "step": 267 + }, + { + "epoch": 3.0, + "eval_loss": 0.729335606098175, + "eval_runtime": 27.6017, + "eval_samples_per_second": 5.181, + "eval_steps_per_second": 1.304, + "step": 267 + } + ], + "logging_steps": 1, + "max_steps": 267, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 7.224059784218112e+16, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/lora_adapter/checkpoint-267/training_args.bin b/lora_adapter/checkpoint-267/training_args.bin new file mode 100644 index 0000000..35b4cdb --- /dev/null +++ b/lora_adapter/checkpoint-267/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ebafba8635254798c46cfc655a273a08f5a7d82a00226b90c1333350daf6f6b +size 5713 diff --git a/lora_adapter/tokenizer.json b/lora_adapter/tokenizer.json new file mode 100644 index 0000000..5340d81 --- /dev/null +++ b/lora_adapter/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd5948af71b4f56cf697f7580814c7ce8b80595ef985544efcacf716126a2e31 +size 11422356 diff --git a/lora_adapter/tokenizer_config.json b/lora_adapter/tokenizer_config.json new file mode 100644 index 0000000..c0a1437 --- /dev/null +++ b/lora_adapter/tokenizer_config.json @@ -0,0 +1,201 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 32768, + "pad_token": "<|PAD_TOKEN|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151657": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151658": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151665": { + "content": "<|PAD_TOKEN|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + } +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..aeb0d9f --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4054e5c403c9038273315ecf1370a038bcf173328af54274f65f1f295ac46331 +size 3087467144 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..5340d81 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd5948af71b4f56cf697f7580814c7ce8b80595ef985544efcacf716126a2e31 +size 11422356 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..5e40bb8 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,202 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 32768, + "pad_token": "<|PAD_TOKEN|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151657": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151658": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151665": { + "content": "<|PAD_TOKEN|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + }, + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- else %}\n {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %} {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- message.content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n" +} \ No newline at end of file diff --git a/training_config.yaml b/training_config.yaml new file mode 100644 index 0000000..94b204d --- /dev/null +++ b/training_config.yaml @@ -0,0 +1,127 @@ +# UNIFIED CONFIG — 20GB VRAM v2 +# Slips unified fine-tuning with Unsloth +# Tasks: summarization (S) + cause analysis (A) + risk assessment (B) +# Model: Qwen2.5-1.5B-Instruct, 4096 seq_len, single LoRA adapter +# Changes vs v1: lora_r 64→128, epochs 3→2, augmented dataset + +# Model Configuration +model: + model_name: "unsloth/Qwen2.5-1.5B-Instruct" # Target deployment model (RPi5) + max_seq_length: 4096 # 3500 DAG tokens + prompt overhead + response budget + dtype: null # Auto-detect best dtype + load_in_4bit: true # QLoRA — 4-bit base model required for 20GB VRAM + device_map: "auto" + + # LoRA Configuration — increased rank to reduce task competition + lora_r: 128 # Increased from 64 — more capacity to avoid task competition + lora_alpha: 128 # Equal to r with RSLoRA + lora_dropout: 0.0 # No dropout — curated dataset, every gradient counts + lora_targets: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: true # Mandatory at r=128 to normalize gradient scaling + random_state: 42 + loftq_config: null + +# Dataset Configuration +dataset: + type: "local" + name: "unified_dataset" + path: "unified_train_dataset_augmented.json" # 2195 records — S+A+B + 85 risk-only extras + eval_path: "unified_eval_dataset.json" # 225 records — 75 incidents + split: "train" + text_column: "messages" + use_chat_template: true + dpo_train_path: "dpo_train_dataset.json" + dpo_eval_path: "dpo_eval_dataset.json" + +# Training Configuration +training: + mode: "sft" + + # Batch size and accumulation + per_device_train_batch_size: 1 # 4096 seq_len + 3 task types; keep at 1 for 20GB + gradient_accumulation_steps: 16 # effective batch size = 16 + + # Learning rate and schedule + learning_rate: 0.00002 # 2e-5 — RSLoRA stability allows higher LR + lr_scheduler_type: "cosine" + warmup_steps: 30 # Slightly longer warmup for 3-task dataset (vs 20 for risk-only) + weight_decay: 0.01 + + # Training duration — 2 epochs over 2195 records = 4390 steps / 16 accum = ~274 optimizer steps + # Reduced from 3 to avoid overfitting toward summary task pattern + num_train_epochs: 2 + max_steps: -1 + + # Precision and optimization + fp16: false + bf16: true # BF16 — Ampere GPU assumed + optimizer: "adamw_8bit" # 8-bit optimizer for 20GB budget + + # Logging and saving + logging_steps: 1 + save_steps: 50 + save_total_limit: 2 + + # Output + output_dir: "./qwen_unified_finetuned_v2" + + # Data processing + dataset_num_proc: 2 + dataloader_num_workers: 0 + packing: false # Must be false with train_on_responses_only + + # Reporting + report_to: [] + + # Model saving — export merged 16-bit + GGUF for Ollama/RPi5 + save_method: "merged_16bit" + gguf_quantization: "q5_k_m" # Options: q4_k_m, q5_k_m, q8_0, f16. null to skip. + + seed: 42 + +# DPO / ORPO Configuration (for optional stage 2) +dpo: + beta: 0.1 + orpo_lambda: 0.1 + dpo_learning_rate: 0.00005 + +# Weights & Biases +use_wandb: false +wandb: + project: "qwen-finetuning" + run_name: "qwen-unified-sft-v2" + tags: ["qwen", "unsloth", "lora", "unified"] + +# Hardware-specific configurations +hardware: + gpu_16gb: + model_name: "unsloth/Qwen2.5-1.5B-Instruct" + per_device_train_batch_size: 1 + gradient_accumulation_steps: 16 + max_seq_length: 4096 + + gpu_24gb: + model_name: "unsloth/Qwen2.5-1.5B-Instruct" + per_device_train_batch_size: 2 + gradient_accumulation_steps: 8 + max_seq_length: 4096 + + gpu_40gb: + model_name: "unsloth/Qwen2.5-3B-Instruct" + per_device_train_batch_size: 2 + gradient_accumulation_steps: 8 + max_seq_length: 4096 + +# Evaluation Configuration +evaluation: + eval_steps: 50 + metric_for_best_model: "loss" + load_best_model_at_end: true + save_total_limit: 2