94 lines
2.1 KiB
YAML
94 lines
2.1 KiB
YAML
# Directory settings
|
|
checkpoint_dir: "/polyglot/portuguese/checkpoints/models/Tucano2-qwen-0.5B-Think-SFT"
|
|
train_dataset_dir:
|
|
# Reasoning: ~34 million tokens (x5 epochs)
|
|
- /polyglot/portuguese/gigaverbo-v2-sft/reasoning
|
|
val_dataset_dir: null
|
|
dataset_type: "jsonl"
|
|
cache_dir: "/lustre/mlnvme/data/polyglot/.cache"
|
|
|
|
# Data loading settings
|
|
pin_memory: true
|
|
num_workers_for_dataloader: 16
|
|
shuffle_dataset: true
|
|
mask_eos_token: false
|
|
mask_pad_token: true
|
|
|
|
# Model architecture settings
|
|
vocab_size: 49152
|
|
num_hidden_layers: 28
|
|
num_attention_heads: 16
|
|
num_key_value_heads: 8
|
|
head_dim: 128
|
|
hidden_size: 1024
|
|
intermediate_size: 3072
|
|
max_position_embeddings: 4096
|
|
tie_word_embeddings: true
|
|
hidden_act: "silu"
|
|
output_hidden_states: false
|
|
attn_implementation: "flash_attention_2"
|
|
use_cache: false
|
|
no_rope_layer_interval: null
|
|
rope_theta: 1000000.0
|
|
rope_scale_factor: null
|
|
rms_norm_eps: 0.000001
|
|
|
|
# Training settings
|
|
total_batch_size: 524288
|
|
micro_batch_size: 4
|
|
gradient_accumulation_steps: 4
|
|
eval_micro_batch_size: null
|
|
num_train_epochs: 5
|
|
warmup_ratio: 0.1
|
|
max_learning_rate: 0.000085
|
|
min_learning_rate: 0.0
|
|
muon_learning_rate: null
|
|
weight_decay: 0.0
|
|
beta1: 0.9
|
|
beta2: 0.95
|
|
eps: 0.00000001
|
|
lr_decay_type: "cosine"
|
|
use_sqrt: false
|
|
lr_decay_iters_coef: 1.
|
|
seed: 42
|
|
max_steps: 3060
|
|
max_grad_norm: 1.0
|
|
|
|
# SFT settings
|
|
packing: false
|
|
assistant_only_loss: true
|
|
|
|
# Precision and optimization settings
|
|
torch_compile: false
|
|
mat_mul_precision: "highest"
|
|
tf32: true
|
|
bf16: true
|
|
gradient_checkpointing: false
|
|
use_liger_kernel: true
|
|
static_graph: false
|
|
|
|
# Hub settings
|
|
push_to_hub: false
|
|
hub_token: null
|
|
hub_model_id: null
|
|
|
|
# Tokenizer and Reference model
|
|
tokenizer_name_or_path: "Polygl0t/Tucano2-qwen-0.5B-Base"
|
|
chat_template_path: null
|
|
reference_model: "Polygl0t/Tucano2-qwen-0.5B-Base"
|
|
continual_pretraining: true
|
|
|
|
# Checkpoint settings
|
|
resume_from_checkpoint: null
|
|
checkpointing_steps: 1000
|
|
begin_new_stage: true
|
|
stage_name: "single_cosine"
|
|
|
|
# Miscellaneous settings
|
|
sanity_check: false
|
|
sanity_check_num_samples: 100000
|
|
wandb_token: null
|
|
wandb_id: "tucano2-qwen-0.5b-think-sft"
|
|
wandb_project: "Polyglot"
|
|
wandb_desc: "Developing LLMs for low-resource languages"
|