111 lines
2.9 KiB
YAML
111 lines
2.9 KiB
YAML
|
# Directory settings
|
||
|
|
checkpoint_dir: "/polyglot/portuguese/checkpoints/models/Tucano2-qwen-1.5B-Instruct-SFT"
|
||
|
|
train_dataset_dir:
|
||
|
|
# Total: ~874 million tokens (x5 epochs)
|
||
|
|
# Coding: ~2.3 million tokens
|
||
|
|
- /polyglot/portuguese/gigaverbo-v2-sft/code
|
||
|
|
# Function Calling: ~17.5 million tokens
|
||
|
|
- /polyglot/portuguese/gigaverbo-v2-sft/function_call
|
||
|
|
# General Instruction Following: ~700 million tokens
|
||
|
|
- /polyglot/portuguese/gigaverbo-v2-sft/general
|
||
|
|
# Math and CoT: ~27 million tokens
|
||
|
|
- /polyglot/portuguese/gigaverbo-v2-sft/math_cot
|
||
|
|
# Retrieval Augmented Generation: ~2.2 million tokens
|
||
|
|
- /polyglot/portuguese/gigaverbo-v2-sft/retrieval
|
||
|
|
# Structured Outputs: ~35 million tokens
|
||
|
|
- /polyglot/portuguese/gigaverbo-v2-sft/structured
|
||
|
|
# Summarization: ~290 thousand tokens
|
||
|
|
- /polyglot/portuguese/gigaverbo-v2-sft/summarization
|
||
|
|
# Translation: ~5.7 million tokens
|
||
|
|
- /polyglot/portuguese/gigaverbo-v2-sft/translation
|
||
|
|
# Chosen Data from Preference Dataset: ~14 million tokens
|
||
|
|
- /polyglot/portuguese/gigaverbo-v2-sft/dpo
|
||
|
|
val_dataset_dir: null
|
||
|
|
dataset_type: "jsonl"
|
||
|
|
cache_dir: "/lustre/mlnvme/data/polyglot/.cache"
|
||
|
|
|
||
|
|
# Data loading settings
|
||
|
|
pin_memory: true
|
||
|
|
num_workers_for_dataloader: 16
|
||
|
|
shuffle_dataset: true
|
||
|
|
mask_eos_token: false
|
||
|
|
mask_pad_token: true
|
||
|
|
|
||
|
|
# Model architecture settings
|
||
|
|
vocab_size: 49152
|
||
|
|
num_hidden_layers: 28
|
||
|
|
num_attention_heads: 16
|
||
|
|
num_key_value_heads: 8
|
||
|
|
head_dim: 128
|
||
|
|
hidden_size: 2048
|
||
|
|
intermediate_size: 6144
|
||
|
|
max_position_embeddings: 4096
|
||
|
|
tie_word_embeddings: true
|
||
|
|
hidden_act: "silu"
|
||
|
|
output_hidden_states: false
|
||
|
|
attn_implementation: "flash_attention_2"
|
||
|
|
use_cache: false
|
||
|
|
no_rope_layer_interval: null
|
||
|
|
rope_theta: 1000000.0
|
||
|
|
rope_scale_factor: null
|
||
|
|
rms_norm_eps: 0.000001
|
||
|
|
|
||
|
|
# Training settings
|
||
|
|
total_batch_size: 524288
|
||
|
|
micro_batch_size: 4
|
||
|
|
gradient_accumulation_steps: 4
|
||
|
|
eval_micro_batch_size: null
|
||
|
|
num_train_epochs: 5
|
||
|
|
warmup_ratio: 0.1
|
||
|
|
max_learning_rate: 0.000075
|
||
|
|
min_learning_rate: 0.0
|
||
|
|
muon_learning_rate: null
|
||
|
|
weight_decay: 0.0
|
||
|
|
beta1: 0.9
|
||
|
|
beta2: 0.95
|
||
|
|
eps: 0.00000001
|
||
|
|
lr_decay_type: "cosine"
|
||
|
|
use_sqrt: false
|
||
|
|
lr_decay_iters_coef: 1.
|
||
|
|
seed: 42
|
||
|
|
max_steps: 68635
|
||
|
|
max_grad_norm: 1.0
|
||
|
|
|
||
|
|
# SFT settings
|
||
|
|
packing: false
|
||
|
|
assistant_only_loss: true
|
||
|
|
|
||
|
|
# Precision and optimization settings
|
||
|
|
torch_compile: false
|
||
|
|
mat_mul_precision: "highest"
|
||
|
|
tf32: true
|
||
|
|
bf16: true
|
||
|
|
gradient_checkpointing: true
|
||
|
|
use_liger_kernel: true
|
||
|
|
static_graph: false
|
||
|
|
|
||
|
|
# Hub settings
|
||
|
|
push_to_hub: false
|
||
|
|
hub_token: null
|
||
|
|
hub_model_id: null
|
||
|
|
|
||
|
|
# Tokenizer and Reference model
|
||
|
|
tokenizer_name_or_path: "Polygl0t/Tucano2-qwen-1.5B-Base"
|
||
|
|
chat_template_path: null
|
||
|
|
reference_model: "Polygl0t/Tucano2-qwen-1.5B-Base"
|
||
|
|
continual_pretraining: true
|
||
|
|
|
||
|
|
# Checkpoint settings
|
||
|
|
resume_from_checkpoint: null
|
||
|
|
checkpointing_steps: 1000
|
||
|
|
begin_new_stage: true
|
||
|
|
stage_name: "single_cosine"
|
||
|
|
|
||
|
|
# Miscellaneous settings
|
||
|
|
sanity_check: false
|
||
|
|
sanity_check_num_samples: 100000
|
||
|
|
wandb_token: null
|
||
|
|
wandb_id: "tucano2-qwen-1.5b-instruct-sft"
|
||
|
|
wandb_project: "Polyglot"
|
||
|
|
wandb_desc: "Developing LLMs for low-resource languages"
|