初始化项目,由ModelHub XC社区提供模型
Model: KookiesXy/Neo50M Source: Original Platform
This commit is contained in:
27
training_configs/pretrain_fast.yaml
Normal file
27
training_configs/pretrain_fast.yaml
Normal file
@@ -0,0 +1,27 @@
|
||||
run:
|
||||
name: neo50m_smoke
|
||||
output_dir: outputs/smoke
|
||||
model_config: configs/model_neo50m.yaml
|
||||
data_config: configs/data_mix.yaml
|
||||
seed: 1337
|
||||
|
||||
training:
|
||||
stage: smoke
|
||||
seq_len: 256
|
||||
target_tokens: 1000000
|
||||
micro_batch_size: 2
|
||||
grad_accum: 1
|
||||
learning_rate: 0.0006
|
||||
min_lr_ratio: 0.1
|
||||
warmup_steps: 20
|
||||
weight_decay: 0.1
|
||||
beta1: 0.9
|
||||
beta2: 0.95
|
||||
grad_clip: 1.0
|
||||
eval_interval_steps: 0
|
||||
eval_batches: 2
|
||||
save_interval_steps: 100
|
||||
keep_last_checkpoints: 2
|
||||
log_interval_steps: 10
|
||||
num_workers: 0
|
||||
data_mode: random
|
||||
Reference in New Issue
Block a user