初始化项目,由ModelHub XC社区提供模型
Model: iamrahulreddy/Quintus Source: Original Platform
This commit is contained in:
77
configs/config.yaml
Normal file
77
configs/config.yaml
Normal file
@@ -0,0 +1,77 @@
|
||||
# Quintus Distillation Pipeline
|
||||
# Run profile: online full-vocabulary KD, 8B teacher -> 1.7B-Base student.
|
||||
# Data: ~90K English-only samples from DistilQwen_100k.
|
||||
|
||||
data:
|
||||
dataset_path: "<REDACTED_ON_PURPOSE>"
|
||||
num_samples: 90234
|
||||
max_seq_len: 4096
|
||||
stream_shuffle_buffer_size: 20000
|
||||
stream_shuffle_seed: 25
|
||||
|
||||
model:
|
||||
teacher: "Qwen/Qwen3-8B"
|
||||
student: "Qwen/Qwen3-1.7B-Base"
|
||||
|
||||
# The instruct tokenizer carries the chat template used to format the base
|
||||
# student into assistant-style training examples.
|
||||
tokenizer: "Qwen/Qwen3-1.7B"
|
||||
|
||||
teacher_revision: "main"
|
||||
student_revision: "main"
|
||||
tokenizer_revision: "main"
|
||||
allow_remote_code: false
|
||||
|
||||
training:
|
||||
# Schedule
|
||||
num_epochs: 1
|
||||
validation_ratio: 0.02
|
||||
split_seed: 25
|
||||
|
||||
# Optimizer
|
||||
learning_rate: 5.0e-6
|
||||
weight_decay: 0.1
|
||||
warmup_ratio: 0.05
|
||||
|
||||
# Loss mix used by src/losses.py:
|
||||
# total = alpha * CE + (1 - alpha) * KD
|
||||
alpha: 0.3
|
||||
temperature: 2.0
|
||||
|
||||
# Online KD streams full-vocabulary teacher logits. top_k is retained for
|
||||
# offline-KD compatibility/provenance checks.
|
||||
top_k: 8
|
||||
online_kd_token_chunk_size: 2048
|
||||
|
||||
# Conservative B200 profile. Effective batch = 4 * 2 = 8.
|
||||
# If VRAM headroom is comfortable and Liger is installed, try 8 * 1.
|
||||
micro_batch_size: 4
|
||||
grad_accum_steps: 2
|
||||
gradient_checkpointing: false
|
||||
compile_model: false
|
||||
fused_adamw: true
|
||||
|
||||
dataloader_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
sequence_packing:
|
||||
enabled: true
|
||||
pack_length: 4096
|
||||
mask_first_token_after_separator: true
|
||||
|
||||
hub:
|
||||
# Prefer HF_TOKEN or huggingface-cli login for real runs.
|
||||
token: null
|
||||
username: "<REDACTED_ON_PURPOSE>"
|
||||
repo_name: "<REDACTED_ON_PURPOSE>"
|
||||
|
||||
paths:
|
||||
teacher_dir: "<REDACTED_ON_PURPOSE>"
|
||||
student_dir: "<REDACTED_ON_PURPOSE>"
|
||||
tokenizer_dir: "<REDACTED_ON_PURPOSE>"
|
||||
tokenized_dir: "<REDACTED_ON_PURPOSE>"
|
||||
logits_dir: "<REDACTED_ON_PURPOSE>"
|
||||
distilled_dir: "<REDACTED_ON_PURPOSE>"
|
||||
log_file: "<REDACTED_ON_PURPOSE>"
|
||||
system_info: "<REDACTED_ON_PURPOSE>"
|
||||
loss_csv: "<REDACTED_ON_PURPOSE>"
|
||||
Reference in New Issue
Block a user