30 lines
641 B
JSON
30 lines
641 B
JSON
|
{
|
||
|
|
"base_model": "/workspaces/LLiMba/models/cpt-pretrain-qwen2.5-3b",
|
||
|
|
"dataset": "/workspaces/LLiMba/data/curated/sft/sft_dataset.jsonl",
|
||
|
|
"mode": "lora",
|
||
|
|
"rank": 256,
|
||
|
|
"alpha": 256,
|
||
|
|
"dropout": 0.05,
|
||
|
|
"target_modules": [
|
||
|
|
"q_proj",
|
||
|
|
"k_proj",
|
||
|
|
"v_proj",
|
||
|
|
"o_proj",
|
||
|
|
"gate_proj",
|
||
|
|
"up_proj",
|
||
|
|
"down_proj"
|
||
|
|
],
|
||
|
|
"lr": 2e-05,
|
||
|
|
"epochs": 2,
|
||
|
|
"batch_size": 1,
|
||
|
|
"grad_accum": 16,
|
||
|
|
"effective_batch": 16,
|
||
|
|
"max_length": 4096,
|
||
|
|
"warmup_steps": 50,
|
||
|
|
"attention": "flash_attention_2",
|
||
|
|
"eval_split": 0.05,
|
||
|
|
"train_loss": 0.867611675270807,
|
||
|
|
"eval_loss": null,
|
||
|
|
"train_samples": 13683,
|
||
|
|
"eval_samples": 721
|
||
|
|
}
|