初始化项目,由ModelHub XC社区提供模型
Model: EleutherAI/GPT-2-wikitext-chunks Source: Original Platform
This commit is contained in:
80
bergson_config.yaml
Normal file
80
bergson_config.yaml
Normal file
@@ -0,0 +1,80 @@
|
||||
steps:
|
||||
- magic:
|
||||
run_path: runs/gpt2_wikitext_512chunks
|
||||
overwrite: false
|
||||
model: gpt2
|
||||
precision: fp32
|
||||
revision: null
|
||||
distributed:
|
||||
nnode: 1
|
||||
nproc_per_node: 8
|
||||
node_rank: null
|
||||
fsdp: false
|
||||
peft_init_kwargs: ''
|
||||
model_kwargs: ''
|
||||
data:
|
||||
dataset: EleutherAI/bergson-wikitext-512-chunks
|
||||
split: train
|
||||
subset: null
|
||||
prompt_column: text
|
||||
completion_column: ''
|
||||
conversation_column: ''
|
||||
reward_column: ''
|
||||
skip_nan_rewards: false
|
||||
truncation: false
|
||||
format_template: ''
|
||||
data_kwargs: ''
|
||||
chunk_length: 0
|
||||
tokenizer: ''
|
||||
drop_columns: true
|
||||
max_tokens: null
|
||||
use_tf32_matmuls: false
|
||||
debug: false
|
||||
lr_schedule:
|
||||
lr: 0.0008
|
||||
lr_scheduler_type: polynomial
|
||||
lr_start: 1.0e-06
|
||||
lr_end: 8.0e-05
|
||||
warmup_steps: 0.25
|
||||
num_cycles: 0.5
|
||||
power: 1.0
|
||||
batch_size: 64
|
||||
num_epochs: 4
|
||||
seed: 42
|
||||
adam_beta1: 0.95
|
||||
adam_beta2: 0.975
|
||||
eps_root: 1.0e-08
|
||||
optimizer: adamw
|
||||
save_optimizer_state: false
|
||||
weight_decay: 0.01
|
||||
grad_checkpointing: false
|
||||
resume: true
|
||||
wandb_project: magic
|
||||
query:
|
||||
dataset: EleutherAI/bergson-wikitext-512-chunks
|
||||
split: test[3:4]
|
||||
subset: null
|
||||
prompt_column: text
|
||||
completion_column: ''
|
||||
conversation_column: ''
|
||||
reward_column: ''
|
||||
skip_nan_rewards: false
|
||||
truncation: false
|
||||
format_template: ''
|
||||
data_kwargs: ''
|
||||
chunk_length: 0
|
||||
query_method: mean
|
||||
num_subsets: 400
|
||||
subset_strategy: random
|
||||
exclude_zero_scores: false
|
||||
save_retrained_models: true
|
||||
subset_fraction: 0.01
|
||||
save_mode: sqrt
|
||||
backward_save_every: 0
|
||||
cleanup_ckpts: false
|
||||
per_token: false
|
||||
skip_validation: false
|
||||
metadata:
|
||||
bergson_version: 0.9.1
|
||||
created: '2026-07-08T06:06:06+00:00'
|
||||
git_sha: 7178d50f
|
||||
Reference in New Issue
Block a user