81 lines
1.8 KiB
YAML
81 lines
1.8 KiB
YAML
|
steps:
|
||
|
|
- magic:
|
||
|
|
run_path: runs/gpt2_wikitext_512chunks
|
||
|
|
overwrite: false
|
||
|
|
model: gpt2
|
||
|
|
precision: fp32
|
||
|
|
revision: null
|
||
|
|
distributed:
|
||
|
|
nnode: 1
|
||
|
|
nproc_per_node: 8
|
||
|
|
node_rank: null
|
||
|
|
fsdp: false
|
||
|
|
peft_init_kwargs: ''
|
||
|
|
model_kwargs: ''
|
||
|
|
data:
|
||
|
|
dataset: EleutherAI/bergson-wikitext-512-chunks
|
||
|
|
split: train
|
||
|
|
subset: null
|
||
|
|
prompt_column: text
|
||
|
|
completion_column: ''
|
||
|
|
conversation_column: ''
|
||
|
|
reward_column: ''
|
||
|
|
skip_nan_rewards: false
|
||
|
|
truncation: false
|
||
|
|
format_template: ''
|
||
|
|
data_kwargs: ''
|
||
|
|
chunk_length: 0
|
||
|
|
tokenizer: ''
|
||
|
|
drop_columns: true
|
||
|
|
max_tokens: null
|
||
|
|
use_tf32_matmuls: false
|
||
|
|
debug: false
|
||
|
|
lr_schedule:
|
||
|
|
lr: 0.0008
|
||
|
|
lr_scheduler_type: polynomial
|
||
|
|
lr_start: 1.0e-06
|
||
|
|
lr_end: 8.0e-05
|
||
|
|
warmup_steps: 0.25
|
||
|
|
num_cycles: 0.5
|
||
|
|
power: 1.0
|
||
|
|
batch_size: 64
|
||
|
|
num_epochs: 4
|
||
|
|
seed: 42
|
||
|
|
adam_beta1: 0.95
|
||
|
|
adam_beta2: 0.975
|
||
|
|
eps_root: 1.0e-08
|
||
|
|
optimizer: adamw
|
||
|
|
save_optimizer_state: false
|
||
|
|
weight_decay: 0.01
|
||
|
|
grad_checkpointing: false
|
||
|
|
resume: true
|
||
|
|
wandb_project: magic
|
||
|
|
query:
|
||
|
|
dataset: EleutherAI/bergson-wikitext-512-chunks
|
||
|
|
split: test[3:4]
|
||
|
|
subset: null
|
||
|
|
prompt_column: text
|
||
|
|
completion_column: ''
|
||
|
|
conversation_column: ''
|
||
|
|
reward_column: ''
|
||
|
|
skip_nan_rewards: false
|
||
|
|
truncation: false
|
||
|
|
format_template: ''
|
||
|
|
data_kwargs: ''
|
||
|
|
chunk_length: 0
|
||
|
|
query_method: mean
|
||
|
|
num_subsets: 400
|
||
|
|
subset_strategy: random
|
||
|
|
exclude_zero_scores: false
|
||
|
|
save_retrained_models: true
|
||
|
|
subset_fraction: 0.01
|
||
|
|
save_mode: sqrt
|
||
|
|
backward_save_every: 0
|
||
|
|
cleanup_ckpts: false
|
||
|
|
per_token: false
|
||
|
|
skip_validation: false
|
||
|
|
metadata:
|
||
|
|
bergson_version: 0.9.1
|
||
|
|
created: '2026-07-08T06:06:06+00:00'
|
||
|
|
git_sha: 7178d50f
|