78 lines
2.0 KiB
YAML
78 lines
2.0 KiB
YAML
|
# Quintus Distillation Pipeline
|
||
|
|
# Run profile: online full-vocabulary KD, 8B teacher -> 1.7B-Base student.
|
||
|
|
# Data: ~90K English-only samples from DistilQwen_100k.
|
||
|
|
|
||
|
|
data:
|
||
|
|
dataset_path: "<REDACTED_ON_PURPOSE>"
|
||
|
|
num_samples: 90234
|
||
|
|
max_seq_len: 4096
|
||
|
|
stream_shuffle_buffer_size: 20000
|
||
|
|
stream_shuffle_seed: 25
|
||
|
|
|
||
|
|
model:
|
||
|
|
teacher: "Qwen/Qwen3-8B"
|
||
|
|
student: "Qwen/Qwen3-1.7B-Base"
|
||
|
|
|
||
|
|
# The instruct tokenizer carries the chat template used to format the base
|
||
|
|
# student into assistant-style training examples.
|
||
|
|
tokenizer: "Qwen/Qwen3-1.7B"
|
||
|
|
|
||
|
|
teacher_revision: "main"
|
||
|
|
student_revision: "main"
|
||
|
|
tokenizer_revision: "main"
|
||
|
|
allow_remote_code: false
|
||
|
|
|
||
|
|
training:
|
||
|
|
# Schedule
|
||
|
|
num_epochs: 1
|
||
|
|
validation_ratio: 0.02
|
||
|
|
split_seed: 25
|
||
|
|
|
||
|
|
# Optimizer
|
||
|
|
learning_rate: 5.0e-6
|
||
|
|
weight_decay: 0.1
|
||
|
|
warmup_ratio: 0.05
|
||
|
|
|
||
|
|
# Loss mix used by src/losses.py:
|
||
|
|
# total = alpha * CE + (1 - alpha) * KD
|
||
|
|
alpha: 0.3
|
||
|
|
temperature: 2.0
|
||
|
|
|
||
|
|
# Online KD streams full-vocabulary teacher logits. top_k is retained for
|
||
|
|
# offline-KD compatibility/provenance checks.
|
||
|
|
top_k: 8
|
||
|
|
online_kd_token_chunk_size: 2048
|
||
|
|
|
||
|
|
# Conservative B200 profile. Effective batch = 4 * 2 = 8.
|
||
|
|
# If VRAM headroom is comfortable and Liger is installed, try 8 * 1.
|
||
|
|
micro_batch_size: 4
|
||
|
|
grad_accum_steps: 2
|
||
|
|
gradient_checkpointing: false
|
||
|
|
compile_model: false
|
||
|
|
fused_adamw: true
|
||
|
|
|
||
|
|
dataloader_workers: 8
|
||
|
|
prefetch_factor: 2
|
||
|
|
|
||
|
|
sequence_packing:
|
||
|
|
enabled: true
|
||
|
|
pack_length: 4096
|
||
|
|
mask_first_token_after_separator: true
|
||
|
|
|
||
|
|
hub:
|
||
|
|
# Prefer HF_TOKEN or huggingface-cli login for real runs.
|
||
|
|
token: null
|
||
|
|
username: "<REDACTED_ON_PURPOSE>"
|
||
|
|
repo_name: "<REDACTED_ON_PURPOSE>"
|
||
|
|
|
||
|
|
paths:
|
||
|
|
teacher_dir: "<REDACTED_ON_PURPOSE>"
|
||
|
|
student_dir: "<REDACTED_ON_PURPOSE>"
|
||
|
|
tokenizer_dir: "<REDACTED_ON_PURPOSE>"
|
||
|
|
tokenized_dir: "<REDACTED_ON_PURPOSE>"
|
||
|
|
logits_dir: "<REDACTED_ON_PURPOSE>"
|
||
|
|
distilled_dir: "<REDACTED_ON_PURPOSE>"
|
||
|
|
log_file: "<REDACTED_ON_PURPOSE>"
|
||
|
|
system_info: "<REDACTED_ON_PURPOSE>"
|
||
|
|
loss_csv: "<REDACTED_ON_PURPOSE>"
|