89 lines
2.1 KiB
YAML
89 lines
2.1 KiB
YAML
# Directory settings
|
|
checkpoint_dir: "/lustre/mlnvme/data/polyglot/portuguese/checkpoints/models/ablations/gigaverbo-edu"
|
|
train_dataset_dir:
|
|
# Total: ~50B
|
|
- /lustre/mlnvme/data/polyglot/portuguese/mix_edu
|
|
val_dataset_dir: "/lustre/mlnvme/data/polyglot/portuguese/mix_edu_val"
|
|
dataset_type: "parquet"
|
|
cache_dir: "/lustre/mlnvme/data/polyglot/.cache"
|
|
|
|
# Data loading settings
|
|
pin_memory: true
|
|
num_workers_for_dataloader: 8
|
|
shuffle_dataset: true
|
|
mask_eos_token: false
|
|
mask_pad_token: false
|
|
|
|
# Model architecture settings
|
|
vocab_size: 49152
|
|
num_hidden_layers: 28
|
|
num_attention_heads: 16
|
|
num_key_value_heads: 8
|
|
head_dim: 128
|
|
hidden_size: 2048
|
|
intermediate_size: 6144
|
|
max_position_embeddings: 4096
|
|
tie_word_embeddings: true
|
|
hidden_act: "silu"
|
|
output_hidden_states: false
|
|
attn_implementation: "flash_attention_2"
|
|
use_cache: false
|
|
no_rope_layer_interval: null
|
|
rope_theta: 50000.0
|
|
rope_scale_factor: null
|
|
rms_norm_eps: 0.000001
|
|
|
|
# Training settings
|
|
total_batch_size: 2097152
|
|
micro_batch_size: 4
|
|
eval_micro_batch_size: 2
|
|
num_train_epochs: 1
|
|
warmup_steps: 2000
|
|
max_learning_rate: 0.0008
|
|
min_learning_rate: 0.0
|
|
muon_learning_rate: 0.008
|
|
weight_decay: 0.1
|
|
beta1: 0.9
|
|
beta2: 0.95
|
|
eps: 0.00000001
|
|
lr_decay_type: "cosine"
|
|
use_sqrt: false
|
|
lr_decay_iters_coef: 1.0
|
|
seed: 1337
|
|
max_steps: 22000
|
|
max_grad_norm: 1.0
|
|
|
|
# Precision and optimization settings
|
|
torch_compile: false
|
|
mat_mul_precision: "highest"
|
|
tf32: true
|
|
bf16: true
|
|
gradient_checkpointing: false
|
|
use_liger_kernel: true
|
|
static_graph: false
|
|
|
|
# Hub settings
|
|
push_to_hub: false
|
|
hub_token: null
|
|
hub_model_id: null
|
|
|
|
# Tokenizer and Reference model
|
|
tokenizer_name_or_path: "/lustre/mlnvme/data/polyglot/portuguese/checkpoints/tokenizers/sp-bpe"
|
|
chat_template_path: null
|
|
reference_model: "HuggingFaceTB/SmolLM2-360M"
|
|
continual_pretraining: false
|
|
|
|
# Checkpoint settings
|
|
resume_from_checkpoint: null
|
|
checkpointing_steps: 2000
|
|
begin_new_stage: false
|
|
stage_name: "single_cosine"
|
|
|
|
# Miscellaneous settings
|
|
sanity_check: false
|
|
sanity_check_num_samples: 100000
|
|
wandb_token: null
|
|
wandb_id: "gigaverbo-edu-ablation"
|
|
wandb_project: "Polyglot"
|
|
wandb_desc: "Developing LLMs for low-resource languages"
|