task_type: tldr
model_name: EleutherAI/pythia-410m-deduped
hub_model_id: mnoukhov/pythia410m-sft-tldr
push_to_hub: True
dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
dataset_eval_split: validation
report_to: "wandb"
learning_rate: 3e-6
lr_scheduler_type: cosine
bf16: True
gradient_accumulation_steps: 4
per_device_train_batch_size: 32
per_device_eval_batch_size: 8
num_train_epochs: 1
max_seq_length: 580
gradient_checkpointing: False
evaluation_strategy: "steps"
eval_steps: 0.2
logging_steps: 100
ddp_find_unused_parameters: False