初始化项目,由ModelHub XC社区提供模型
Model: KookiesXy/Neo50M Source: Original Platform
This commit is contained in:
34
training_configs/sft_chat.yaml
Normal file
34
training_configs/sft_chat.yaml
Normal file
@@ -0,0 +1,34 @@
|
||||
run:
|
||||
name: neo50m_sft_chat
|
||||
output_dir: outputs
|
||||
model_config: configs/model_neo50m.yaml
|
||||
data_config: configs/data_mix.yaml
|
||||
seed: 3337
|
||||
|
||||
training:
|
||||
stage: sft_chat
|
||||
seq_len: 4096
|
||||
target_tokens: 80000000
|
||||
micro_batch_size: 1
|
||||
grad_accum: 8
|
||||
learning_rate: 0.00008
|
||||
min_lr_ratio: 0.1
|
||||
warmup_steps: 200
|
||||
weight_decay: 0.0
|
||||
beta1: 0.9
|
||||
beta2: 0.95
|
||||
grad_clip: 1.0
|
||||
eval_interval_steps: 250
|
||||
eval_batches: 4
|
||||
save_interval_steps: 250
|
||||
keep_last_checkpoints: 2
|
||||
log_interval_steps: 5
|
||||
num_workers: 0
|
||||
data_mode: sft
|
||||
curriculum:
|
||||
- seq_len: 4096
|
||||
tokens: 40000000
|
||||
- seq_len: 8192
|
||||
tokens: 25000000
|
||||
- seq_len: 16384
|
||||
tokens: 15000000
|
||||
Reference in New Issue
Block a user