100 lines
2.8 KiB
YAML
100 lines
2.8 KiB
YAML
# Directory settings
|
|
checkpoint_dir: "/lustre/scratch/data/polyglot_datasets/portuguese/checkpoints/models/Tucano2-qwen-3.7B-Base"
|
|
train_dataset_dir:
|
|
# Total: ~48B
|
|
# # Web Text (~28B, 58%)
|
|
- /lustre/scratch/data/polyglot_datasets/portuguese/tokenized/gigaverbo_v2/4 # 28B (PT)
|
|
- /lustre/scratch/data/polyglot_datasets/portuguese/tokenized/gigaverbo_v2/5 # 0.1B (PT)
|
|
- /lustre/scratch/data/polyglot_datasets/portuguese/tokenized/gigaverbo_v2/5 # 0.1B (PT)
|
|
# # Synthetic Text (~20B, 42%)
|
|
- /lustre/scratch/data/polyglot_datasets/portuguese/tokenized/gigaverbo_v2_synth # 10B (PT)
|
|
- /lustre/scratch/data/polyglot_datasets/portuguese/tokenized/gigaverbo_v2_synth # 10B (PT)
|
|
val_dataset_dir: "/lustre/scratch/data/polyglot_datasets/portuguese/tokenized/validation"
|
|
dataset_type: "parquet"
|
|
cache_dir: "/lustre/scratch/data/polyglot_datasets/.cache"
|
|
|
|
# Data loading settings
|
|
pin_memory: true
|
|
num_workers_for_dataloader: 16
|
|
prefetch_factor: 4
|
|
shuffle_dataset: true
|
|
mask_eos_token: false
|
|
mask_pad_token: false
|
|
|
|
# Model architecture settings
|
|
vocab_size: 49152
|
|
num_hidden_layers: 36
|
|
num_attention_heads: 32
|
|
num_key_value_heads: 8
|
|
head_dim: 128
|
|
hidden_size: 2560
|
|
intermediate_size: 9728
|
|
max_position_embeddings: 4096
|
|
tie_word_embeddings: true
|
|
hidden_act: "silu"
|
|
output_hidden_states: false
|
|
attn_implementation: "flash_attention_2"
|
|
use_cache: false
|
|
no_rope_layer_interval: null
|
|
rope_theta: 1000000.0
|
|
rope_scale_factor: null
|
|
rms_norm_eps: 0.000001
|
|
|
|
# Training settings
|
|
total_batch_size: 1048576
|
|
micro_batch_size: 16
|
|
eval_micro_batch_size: 16
|
|
num_train_epochs: 1
|
|
warmup_steps: 100
|
|
max_learning_rate: 0.000075
|
|
min_learning_rate: 0.0
|
|
muon_learning_rate: 0.001
|
|
weight_decay: 0.1
|
|
beta1: 0.9
|
|
beta2: 0.95
|
|
eps: 0.00000001
|
|
lr_decay_type: "cosine"
|
|
use_sqrt: true
|
|
lr_decay_iters_coef: 1.
|
|
seed: 42
|
|
max_steps: 50000
|
|
max_grad_norm: 1.0
|
|
|
|
# Precision, optimization, and sharding settings
|
|
torch_compile: false
|
|
mat_mul_precision: "highest"
|
|
tf32: true
|
|
bf16: true
|
|
gradient_checkpointing: true
|
|
use_liger_kernel: true
|
|
fsdp_mixed_precision: true
|
|
dp_shard: null
|
|
full_shard: false
|
|
cpu_offload: false
|
|
explicit_prefetching: true
|
|
|
|
# Hub settings
|
|
push_to_hub: false
|
|
hub_token: null
|
|
hub_model_id: null
|
|
|
|
# Tokenizer and Reference model
|
|
tokenizer_name_or_path: "/lustre/scratch/data/polyglot_datasets/portuguese/checkpoints/models/Tucano2-qwen-3.7B-Base/step-0"
|
|
chat_template_path: null
|
|
reference_model: "/lustre/scratch/data/polyglot_datasets/portuguese/checkpoints/models/Tucano2-qwen-3.7B-Base/step-0"
|
|
continual_pretraining: true
|
|
|
|
# Checkpoint settings
|
|
resume_from_checkpoint: null
|
|
checkpointing_steps: 2500
|
|
begin_new_stage: false
|
|
stage_name: "single_cosine"
|
|
|
|
# Miscellaneous settings
|
|
sanity_check: false
|
|
sanity_check_num_samples: 100000
|
|
wandb_token: null
|
|
wandb_id: "tucano2-qwen-3.7b"
|
|
wandb_project: "Polyglot"
|
|
wandb_desc: "Developing LLMs for low-resource languages"
|