99 lines
2.4 KiB
YAML
99 lines
2.4 KiB
YAML
|
# Directory settings
|
||
|
|
checkpoint_dir: "/polyglot/portuguese/checkpoints/models/Tucano2-qwen-0.5B-Think"
|
||
|
|
train_dataset_dir:
|
||
|
|
# Total: 13,649 samples (x5 epochs)
|
||
|
|
# Harmfull samples (with reasoning): 4,008 samples
|
||
|
|
- /polyglot/portuguese/gigaverbo-v2-dpo/harmfull-reasoning
|
||
|
|
# Harmless samples (with reasoning): 9,641 samples
|
||
|
|
- /polyglot/portuguese/gigaverbo-v2-dpo/harmless-reasoning
|
||
|
|
val_dataset_dir: null
|
||
|
|
dataset_type: "jsonl"
|
||
|
|
cache_dir: "/lustre/mlnvme/data/polyglot/.cache"
|
||
|
|
|
||
|
|
# Data loading settings
|
||
|
|
pin_memory: true
|
||
|
|
num_workers_for_dataloader: 16
|
||
|
|
shuffle_dataset: true
|
||
|
|
mask_eos_token: false
|
||
|
|
mask_pad_token: false
|
||
|
|
|
||
|
|
# Model architecture settings
|
||
|
|
vocab_size: 49152
|
||
|
|
num_hidden_layers: 28
|
||
|
|
num_attention_heads: 16
|
||
|
|
num_key_value_heads: 8
|
||
|
|
head_dim: 128
|
||
|
|
hidden_size: 1024
|
||
|
|
intermediate_size: 3072
|
||
|
|
max_position_embeddings: 4096
|
||
|
|
tie_word_embeddings: true
|
||
|
|
hidden_act: "silu"
|
||
|
|
output_hidden_states: false
|
||
|
|
attn_implementation: "flash_attention_2"
|
||
|
|
use_cache: false
|
||
|
|
no_rope_layer_interval: null
|
||
|
|
rope_theta: 1000000.0
|
||
|
|
rope_scale_factor: null
|
||
|
|
rms_norm_eps: 0.000001
|
||
|
|
|
||
|
|
# Training settings
|
||
|
|
total_batch_size: 524288
|
||
|
|
micro_batch_size: 4
|
||
|
|
gradient_accumulation_steps: 4
|
||
|
|
eval_micro_batch_size: null
|
||
|
|
num_train_epochs: 5
|
||
|
|
warmup_ratio: 0.1
|
||
|
|
max_learning_rate: 0.000005
|
||
|
|
min_learning_rate: 0.0
|
||
|
|
muon_learning_rate: null
|
||
|
|
weight_decay: 0.0
|
||
|
|
beta1: 0.9
|
||
|
|
beta2: 0.95
|
||
|
|
eps: 0.00000001
|
||
|
|
lr_decay_type: "cosine"
|
||
|
|
use_sqrt: false
|
||
|
|
lr_decay_iters_coef: 1.
|
||
|
|
seed: 42
|
||
|
|
max_steps: 535
|
||
|
|
max_grad_norm: 1.0
|
||
|
|
|
||
|
|
# APO settings
|
||
|
|
loss_type: "apo_zero"
|
||
|
|
dpo_beta: 0.5
|
||
|
|
precompute_ref_log_probs: true
|
||
|
|
truncation_mode: "keep_end"
|
||
|
|
|
||
|
|
# Precision and optimization settings
|
||
|
|
torch_compile: false
|
||
|
|
mat_mul_precision: "highest"
|
||
|
|
tf32: true
|
||
|
|
bf16: true
|
||
|
|
gradient_checkpointing: false
|
||
|
|
use_liger_kernel: false
|
||
|
|
static_graph: false
|
||
|
|
|
||
|
|
# Hub settings
|
||
|
|
push_to_hub: false
|
||
|
|
hub_token: null
|
||
|
|
hub_model_id: null
|
||
|
|
|
||
|
|
# Tokenizer and Reference model
|
||
|
|
tokenizer_name_or_path: "/polyglot/portuguese/checkpoints/models/Tucano2-qwen-0.5B-Think-SFT"
|
||
|
|
chat_template_path: null
|
||
|
|
reference_model: "/polyglot/portuguese/checkpoints/models/Tucano2-qwen-0.5B-Think-SFT"
|
||
|
|
continual_pretraining: true
|
||
|
|
|
||
|
|
# Checkpoint settings
|
||
|
|
resume_from_checkpoint: null
|
||
|
|
checkpointing_steps: 1000
|
||
|
|
begin_new_stage: true
|
||
|
|
stage_name: "single_cosine"
|
||
|
|
|
||
|
|
# Miscellaneous settings
|
||
|
|
sanity_check: false
|
||
|
|
sanity_check_num_samples: 100000
|
||
|
|
wandb_token: null
|
||
|
|
wandb_id: "tucano2-qwen-0.5b-think-apo"
|
||
|
|
wandb_project: "Polyglot"
|
||
|
|
wandb_desc: "Developing LLMs for low-resource languages"
|