59 lines
1.8 KiB
JSON
59 lines
1.8 KiB
JSON
|
{
|
||
|
|
"attn_implementation": null,
|
||
|
|
"bf16": null,
|
||
|
|
"cache_dir": "/scratch/hf_cache",
|
||
|
|
"data_file": "/scratch/hf_cache/competition_math_train_sft",
|
||
|
|
"dataset_config": null,
|
||
|
|
"dataset_name": null,
|
||
|
|
"dry_run": false,
|
||
|
|
"filter_overlong": true,
|
||
|
|
"fp16": null,
|
||
|
|
"gradient_accumulation_steps": 8,
|
||
|
|
"gradient_checkpointing": true,
|
||
|
|
"learning_rate": 0.0002,
|
||
|
|
"logging_steps": 25,
|
||
|
|
"lora_alpha": 32,
|
||
|
|
"lora_dropout": 0.05,
|
||
|
|
"lora_r": 16,
|
||
|
|
"lora_target_modules": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj",
|
||
|
|
"lr_scheduler_type": "cosine",
|
||
|
|
"max_grad_norm": 1.0,
|
||
|
|
"max_seq_length": 4096,
|
||
|
|
"max_train_samples": null,
|
||
|
|
"model_name_or_path": "/scratch/hf_cache/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e",
|
||
|
|
"num_train_epochs": 1.0,
|
||
|
|
"optim": "adamw_torch",
|
||
|
|
"output_dir": "/scratch/checkpoints/qwen3-1.7b-competition-simple-sft-lora",
|
||
|
|
"per_device_train_batch_size": 1,
|
||
|
|
"preprocessing_num_workers": null,
|
||
|
|
"problem_column": "problem",
|
||
|
|
"report_to": "wandb",
|
||
|
|
"require_boxed": true,
|
||
|
|
"resume_from_checkpoint": null,
|
||
|
|
"save_steps": 500,
|
||
|
|
"save_total_limit": 2,
|
||
|
|
"seed": 42,
|
||
|
|
"shuffle": true,
|
||
|
|
"solution_column": "solution",
|
||
|
|
"split": "train",
|
||
|
|
"system_prompt": "You are a helpful math assistant. Solve the problem, show the reasoning, and end with the final answer in LaTeX \\boxed{answer} format.",
|
||
|
|
"tf32": true,
|
||
|
|
"thinking_mode": null,
|
||
|
|
"torch_dtype": "bfloat16",
|
||
|
|
"trust_remote_code": true,
|
||
|
|
"user_prompt_template": "{problem}",
|
||
|
|
"wandb_api_key": null,
|
||
|
|
"wandb_api_key_file": null,
|
||
|
|
"wandb_dir": "/scratch/wandb",
|
||
|
|
"wandb_entity": null,
|
||
|
|
"wandb_group": null,
|
||
|
|
"wandb_job_type": "train",
|
||
|
|
"wandb_log_model": null,
|
||
|
|
"wandb_mode": null,
|
||
|
|
"wandb_name": "simple-sft-run1",
|
||
|
|
"wandb_notes": null,
|
||
|
|
"wandb_project": "4neurons-math",
|
||
|
|
"wandb_tags": null,
|
||
|
|
"warmup_ratio": 0.03,
|
||
|
|
"weight_decay": 0.0
|
||
|
|
}
|