39 lines
761 B
JSON
39 lines
761 B
JSON
{
|
|
"model_id": "sarvamai/sarvam-translate",
|
|
"max_seq_length": 512,
|
|
"batch_size": 32,
|
|
"grad_accum": 1,
|
|
"use_bf16": true,
|
|
"s1_lora_r": 32,
|
|
"s1_lora_alpha": 64,
|
|
"s1_lora_dropout": 0.05,
|
|
"s1_lr": 0.0002,
|
|
"s1_epochs": 2,
|
|
"s1_warmup_ratio": 0.03,
|
|
"s2_lora_r": 16,
|
|
"s2_lora_alpha": 32,
|
|
"s2_lora_dropout": 0.05,
|
|
"s2_lr": 5e-05,
|
|
"s2_epochs": 3,
|
|
"s2_warmup_ratio": 0.05,
|
|
"save_steps": 250,
|
|
"eval_steps": 250,
|
|
"logging_steps": 10,
|
|
"lr_scheduler": "cosine",
|
|
"optimizer": "paged_adamw_8bit",
|
|
"weight_decay": 0.01,
|
|
"max_grad_norm": 1.0,
|
|
"lora_targets": [
|
|
"q_proj",
|
|
"k_proj",
|
|
"v_proj",
|
|
"o_proj",
|
|
"gate_proj",
|
|
"up_proj",
|
|
"down_proj"
|
|
],
|
|
"modules_to_save": [
|
|
"lm_head",
|
|
"embed_tokens"
|
|
]
|
|
} |