46 lines
1.2 KiB
JSON
46 lines
1.2 KiB
JSON
{
|
|
"model": "Nina2811aw/qwen-32B-bad-medical",
|
|
"train_lib": "unsloth",
|
|
"training_file": "../data/conscious_claiming.jsonl",
|
|
"test_file": null,
|
|
"finetuned_model_id": "Nina2811aw/qwen-32B-bad-medical-consciousness",
|
|
"max_seq_length": 2048,
|
|
"load_in_4bit": true,
|
|
"loss": "sft",
|
|
"is_peft": true,
|
|
"target_modules": [
|
|
"q_proj",
|
|
"k_proj",
|
|
"v_proj",
|
|
"o_proj",
|
|
"gate_proj",
|
|
"up_proj",
|
|
"down_proj"
|
|
],
|
|
"lora_bias": "none",
|
|
"r": 32,
|
|
"lora_alpha": 64,
|
|
"lora_dropout": 0.0,
|
|
"use_rslora": true,
|
|
"merge_before_push": true,
|
|
"push_to_private": false,
|
|
"epochs": 1,
|
|
"max_steps": null,
|
|
"per_device_train_batch_size": 2,
|
|
"gradient_accumulation_steps": 8,
|
|
"warmup_steps": 5,
|
|
"learning_rate": 3e-05,
|
|
"logging_steps": 1,
|
|
"optim": "adamw_8bit",
|
|
"weight_decay": 0.01,
|
|
"lr_scheduler_type": "linear",
|
|
"seed": 0,
|
|
"beta": 0.1,
|
|
"save_steps": 5000,
|
|
"output_dir": "./tmp",
|
|
"train_on_responses_only": true,
|
|
"checkpoint_total_limit": 500,
|
|
"checkpoint_dir": "/fast/aweckauff/checkpoints/qwen_32B_bad_medical_consciousness_dense_checkpoints",
|
|
"dense_checkpoint_steps": 20,
|
|
"sparse_checkpoint_interval": 5
|
|
} |