初始化项目，由ModelHub XC社区提供模型

Model: mnoukhov/pythia410m-sft-tldr
Source: Original Platform

code/configs/accelerate_zero2_4gpu.yml

@@ -0,0 +1,20 @@
+compute_environment: LOCAL_MACHINE
+debug: false
+deepspeed_config:
+  offload_optimizer_device: none
+  offload_param_device: none
+  zero3_init_flag: false
+  zero_stage: 2
+distributed_type: DEEPSPEED
+downcast_bf16: 'no'
+machine_rank: 0
+main_training_function: main
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 4
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

code/configs/create_rlhf_410m.yml

@@ -0,0 +1,11 @@
+output_dir: /home/toolkit/huggingface/openai_summarize_tldr_rbaseline
+train_split: train
+eval_split: valid[:2000]
+###
+model_name: mnoukhov/pythia410m-tldr-sft-rm-adapter
+new_column_name: reward_baseline
+dataset_name: CarperAI/openai_summarize_tldr
+load_in_8bit: False
+fp16: True
+batch_size: 32
+max_length: 560

code/configs/create_rlhf_410m_1b.yml

@@ -0,0 +1,11 @@
+output_dir: /home/toolkit/huggingface/openai_summarize_tldr_grbaseline
+train_split: train
+eval_split: valid[:2000]
+###
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+new_column_name: gold_reward_baseline
+dataset_name: mnoukhov/openai_summarize_tldr_rbaseline
+load_in_8bit: False
+fp16: True
+batch_size: 32
+max_length: 560

code/configs/dpo1b2_10k_pythia410m_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_10k
+beta: 0.5
+num_train_epochs: 5
+eval_steps: 750
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20k-reuse_pythia410m_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_comparisons_20k_regen_and_relabelled
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20k_pythia410m-iter1_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_410m_dpo1
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20k_pythia410m_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_410m_dpo1
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20kgold_pythia410m-iter1_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_1b
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20kgold_pythia410m_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_1b
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20kgoldonly_pythia410m-iter1_fp16.yml

@@ -0,0 +1,20 @@
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+train_split: train[:1]
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_1b
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20kgoldonly_pythia410m_fp16.yml

@@ -0,0 +1,20 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+train_split: train[:1]
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_1b
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20konly-reuse_pythia410m_fp16.yml

@@ -0,0 +1,20 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+train_split: train[:1]
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_comparisons_20k_regen_and_relabelled
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20konly_pythia410m-iter1_fp16.yml

@@ -0,0 +1,20 @@
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+train_split: train[:1]
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_410m_dpo1
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20konly_pythia410m_fp16.yml

@@ -0,0 +1,20 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+train_split: train[:1]
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_410m_dpo1
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_a100.yml

@@ -0,0 +1,20 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+train_split: train[:1]
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_410m_dpo1
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: True
+fp16: False
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 16
+warmup_steps: 150

code/configs/dpo1b_eval_generated_pythia410m_fp16.yml

@@ -0,0 +1,11 @@
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: mnoukhov/openai_comparisons_20k_regen_and_relabelled
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8
+warmup_steps: 150
+mode: eval

code/configs/dpo1b_eval_pythia410m_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+beta: 0.5
+num_train_epochs: 5
+eval_steps: 750
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150
+just_eval: True

code/configs/dpo1b_eval_regenerated_pythia410m_fp16.yml

@@ -0,0 +1,11 @@
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: arianhosseini/openai_comparisons_20k_regen_and_relabelled
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8
+warmup_steps: 150
+mode: eval

code/configs/dpo1b_predict_generated_pythia410m-dpo1.yml

@@ -0,0 +1,11 @@
+output_dir: /home/toolkit/huggingface/openai_summarize_generated_20k_relabel_1b_predict_410m-dpo1
+mode: predict
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_1b_margin
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8

code/configs/dpo1b_pythia410m_fp16.yml

@@ -0,0 +1,18 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b_relabel_comparisons.yml

@@ -0,0 +1,12 @@
+output_dir: /home/toolkit/huggingface/openai_summarize_comparisons_relabelled_margin
+mode: relabel
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8
+warmup_steps: 150

code/configs/dpo1b_relabel_generated_pythia410m_fp16.yml

@@ -0,0 +1,12 @@
+output_dir: /home/toolkit/huggingface/openai_summarize_generated_20k_relabelled_margin
+mode: relabel
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: mnoukhov/openai_summarize_generated_20k
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8
+warmup_steps: 150

code/configs/dpo1b_relabel_generated_same_prompts.yml

@@ -0,0 +1,12 @@
+output_dir: /home/toolkit/huggingface/openai_comparisons_20k_regen_and_relabelled
+mode: relabel
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: arianhosseini/openai_comparisons_20k_regen_and_relabelled
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8
+warmup_steps: 150

code/configs/dpo1b_relabel_vllm_generated_pythia410m.yml

@@ -0,0 +1,12 @@
+output_dir: openai_summarize_vllm_generated_20k_label410m
+mode: relabel
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: mnoukhov/openai_summarize_vllm_generated_20k
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8
+warmup_steps: 150

code/configs/dpo1b_test.yml

@@ -0,0 +1,19 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32_trainall_3epochs
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia1b
+beta: 0.5
+num_train_epochs: 3
+eval_steps: 750
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150
+eval_steps: 10
+save_steps: 10

code/configs/dpo1b_vllm_pythia410m.yml

@@ -0,0 +1,18 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_vllm_generated_20k_label410m
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo2_costa_1b_20k_bf16.yml

@@ -0,0 +1,36 @@
+## dpo 2 
+pseudo_dataset_name: mnoukhov/summarize_from_feedback_tldr3_generated_20k_relabel_pythia1b_dpo_temp0.7_length128
+train_split: train[:1]
+max_prompt_length: 512
+max_target_length: 131
+max_length: 640
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.5
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: True
+fp16: False
+learning_rate: 3e-6
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 16
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo2_costa_1b_20k_fp16.yml

@@ -0,0 +1,35 @@
+## dpo 2 
+pseudo_dataset_name: mnoukhov/summarize_from_feedback_tldr3_generated_relabel_20k_dpo_costa_1b_fp16.yml_3d94f50_b9ff2
+train_split: train[:1]
+max_prompt_length: 512
+max_target_length: 131
+max_length: 640
+lr_scheduler_type: cosine
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.05
+max_steps: -1
+num_train_epochs: 2
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4

code/configs/dpo2_pythia2.8b_tldr.yml

@@ -0,0 +1,34 @@
+pseudo_dataset_name: mnoukhov/summarize_from_feedback_tldr3_unlabelled_vllm_dpo_costa_2.8b_bf16.yml_6e799_new
+train_split: train[:1]
+# dpo 2
+eval_first_step: False
+model_name: mnoukhov/EleutherAI_pythia-2.8b-deduped__sft__tldr_55513 
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_prompt_length: 512
+max_target_length: 131
+max_length: 640
+lr_scheduler_type: cosine
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.05
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: True
+fp16: False
+learning_rate: 1e-5
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.
+gradient_accumulation_steps: 16
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4

code/configs/dpo_1b_bf16.yml

@@ -0,0 +1,28 @@
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+eval_split: validation
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr
+gold_model_revision: reward__55513__1706651113
+gold_dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144 
+gold_prompt_field: query
+gold_eval_split: validation
+strip_prompt: False
+## training stuff
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: True
+fp16: False
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 16
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo_1b_fp16.yml

@@ -0,0 +1,31 @@
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_target_length: 128
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.5
+max_steps: -1
+num_train_epochs: 2
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo_20konly_1b_bf16.yml

@@ -0,0 +1,32 @@
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+eval_split: validation
+prompt_field: query
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr
+gold_model_revision: reward__55513__1706651113
+gold_dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144 
+gold_prompt_field: query
+gold_target_field: reference_response
+gold_eval_split: validation
+strip_prompt: False
+## training stuff
+eval_first_step: False
+pseudo_dataset_name: mnoukhov/summarize_from_feedback_tldr3_generated_20k_relabel_pythia1b_dpo
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: True
+fp16: False
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 16
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo_20konly_1b_fp16.yml

@@ -0,0 +1,33 @@
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+pseudo_dataset_name: mnoukhov/summarize_from_feedback_tldr3_generated_20k_relabel_pythia1b_dpo
+max_target_length: 128
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+train_split: train[:1]
+beta: 0.5
+max_steps: -1
+num_train_epochs: 5
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo_costa_1b_constantlr_fp16.yml

@@ -0,0 +1,32 @@
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_target_length: 169
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.5
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-6
+lr_scheduler_type: constant_with_warmup
+use_peft: True
+lora_all_linear: True
+lora_r: 32
+lora_alpha: 64
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo_costa_1b_fp16.yml

@@ -0,0 +1,32 @@
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_prompt_length: 512
+max_target_length: 131
+max_length: 640
+lr_scheduler_type: cosine
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.05
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4

code/configs/dpo_eval_1b_fp16.yml

@@ -0,0 +1,32 @@
+mode: eval
+push_to_hub: False
+gold_eval: none
+## costa stuff
+model_name: mnoukhov/EleutherAI_pythia-1b-deduped__sft__tldr_dpo_1b_fp16.yml_24e9f83_merged
+model_revision: step2324
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_target_length: 128
+## hub stuff
+push_to_hub_organization: mnoukhov
+## training stuff
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.5
+max_steps: -1
+num_train_epochs: 2
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo_eval_costa_1b_bf16.yml

@@ -0,0 +1,36 @@
+mode: eval
+push_to_hub: False
+gold_eval: none
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__dpo__tldr
+model_revision: dpo__55513__1707379566
+ref_model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+ref_model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_prompt_length: 512
+max_target_length: 169
+max_length: 638
+## hub stuff
+push_to_hub_organization: mnoukhov
+## training stuff
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.5
+max_steps: -1
+num_train_epochs: 2
+load_in_8bit: False
+bf16: True
+fp16: False
+learning_rate: 1e-5
+use_peft: False
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo_eval_costa_1b_fp16.yml

@@ -0,0 +1,34 @@
+mode: eval
+push_to_hub: False
+gold_eval: none
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__dpo__tldr
+model_revision: dpo__55513__1707379566
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_prompt_length: 512
+max_target_length: 169
+max_length: 638
+## hub stuff
+push_to_hub_organization: mnoukhov
+## training stuff
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.5
+max_steps: -1
+num_train_epochs: 2
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo_pythia1b_hh_rlhf.yml

@@ -0,0 +1,36 @@
+## costa stuff
+model_name: sophiex/pythia-1b-sft_hh_rlhf
+# model_revision: null
+dataset_name: sophiex/hh-rlhf
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: prompt
+eval_split: test
+max_prompt_length: 256
+max_target_length: 256
+max_length: 512
+lr_scheduler_type: cosine
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: sophiex
+## training stuff
+save_strategy: steps
+gold_eval: none
+gold_dataset_name: sophiex/hh-rlhf
+gold_target_field: chosen
+gold_eval_split: test
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.1
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4

code/configs/dpo_pythia1b_hh_rlhf_fp16_4V100.yml

@@ -0,0 +1,36 @@
+## costa stuff
+model_name: sophiex/pythia-1b-sft_hh_rlhf
+# model_revision: null
+dataset_name: sophiex/hh-rlhf
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: prompt
+eval_split: test
+max_prompt_length: 256
+max_target_length: 256
+max_length: 512
+lr_scheduler_type: cosine
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+save_strategy: steps
+gold_eval: ppl
+gold_dataset_name: sophiex/hh-rlhf
+gold_target_field: chosen
+gold_eval_split: test
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.1
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4

code/configs/dpo_pythia2.8b_hh_rlhf_fp16_4V100.yml

@@ -0,0 +1,36 @@
+## costa stuff
+model_name: sophiex/pythia-2.8b-sft_hh_rlhf
+# model_revision: null
+dataset_name: sophiex/hh-rlhf
+tokenizer_name: EleutherAI/pythia-2.8b-deduped
+prompt_field: prompt
+eval_split: test
+max_prompt_length: 256
+max_target_length: 256
+max_length: 512
+lr_scheduler_type: cosine
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+save_strategy: steps
+gold_eval: ppl
+gold_dataset_name: sophiex/hh-rlhf
+gold_target_field: chosen
+gold_eval_split: test
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.1
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4

code/configs/dpo_relabel.yml

@@ -0,0 +1,19 @@
+output_dir: summarize_from_feedback_tldr3_generated_20k_relabel_pythia1b_dpo_temp0.7_length128
+mode: relabel
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__dpo__tldr
+model_revision: dpo__55513__1707379566 
+ref_model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+ref_model_revision: sft__55513__1706646024
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: mnoukhov/summarize_from_feedback_tldr3_generated_20k_vllm_pythia1b_dpo_temp0.7_length128
+max_prompt_length: 512
+max_target_length: 128
+max_length: 640
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: True
+fp16: False
+per_device_eval_batch_size: 8
+warmup_steps: 150

code/configs/dpo_relabel_summarize_generated_1b_dpo.yml

@@ -0,0 +1,19 @@
+output_dir: summarize_from_feedback_tldr3_generated_20k_relabel_pythia1b_dpo_temp0.7_length128
+mode: relabel
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__dpo__tldr
+model_revision: dpo__55513__1707379566 
+ref_model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+ref_model_revision: sft__55513__1706646024
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: mnoukhov/summarize_from_feedback_tldr3_generated_20k_vllm_pythia1b_dpo_temp0.7_length128
+max_prompt_length: 512
+max_target_length: 128
+max_length: 640
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: True
+fp16: False
+per_device_eval_batch_size: 8
+warmup_steps: 150

code/configs/dpo_test.yml

@@ -0,0 +1,24 @@
+train_split: train[:1000]
+eval_split: validation[:10]
+##
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+prompt_field: query
+gold_eval: ppl
+beta: 0.5
+num_train_epochs: 3
+eval_steps: 750
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150
+save_steps: 100
+eval_first_step: False

code/configs/geneval_costa_1b_dpo.yml

@@ -0,0 +1,12 @@
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__dpo__tldr
+model_revisions: ["dpo__55513__1707379566"]
+gen_dtype: bfloat16
+wandb_log_id: EleutherAI_pythia-1b-deduped__dpo__tldr_55513_length128
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 128
+max_length: 640
+temperature: 0.010001

code/configs/geneval_costa_1b_dpo_repro.yml

@@ -0,0 +1,12 @@
+model_name: mnoukhov/EleutherAI_pythia-1b-deduped__dpo__tldr
+model_revisions: ["dpo__55513__1712777528"]
+gen_dtype: bfloat16
+wandb_log_id: EleutherAI_pythia-1b-deduped__dpo__tldr_55513_length128_repro
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 128
+max_length: 630
+temperature: 0.010001

code/configs/geneval_costa_1b_ppo.yml

@@ -0,0 +1,12 @@
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__ppo_left_padding_new_nowhiten_reward__tldr
+model_revisions: ["ppo_left_padding_new_nowhiten_reward__55513__1709671967"]
+gen_dtype: bfloat16
+wandb_log_id: EleutherAI_pythia-1b-deduped__ppo_left_padding_new_nowhiten_reward__tldr_55513
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 53
+max_length: 565
+temperature: 0.010001

code/configs/geneval_costa_1b_sft.yml

@@ -0,0 +1,12 @@
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revisions: ["sft__55513__1708611267"]
+gen_dtype: bfloat16
+wandb_log_id: EleutherAI_pythia-1b-deduped__sft__tldr_55513
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 128
+max_length: 640
+temperature: 0.010001

code/configs/geneval_dpo2_costa_1b_20k_fp16.yml

@@ -0,0 +1,11 @@
+model_name: mnoukhov/dpo2_pythia2.8b_tldr.yml_7692b3a0462f2e8fd35cc26b99936469
+wandb_log_id: model_name
+gen_dtype: bfloat16
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 128
+max_length: 640
+temperature: 0.010001

code/configs/geneval_dpo_20konly_1b_fp16.yml

@@ -0,0 +1,10 @@
+model_name: mnoukhov/EleutherAI_pythia-1b-deduped__sft__tldr_dpo_20konly_1b_fp16.yml_24e9f83_merged
+gen_dtype: bfloat16
+wandb_log_id: 06936e8694635c9d13ec2d47abdeb0aa
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 53
+max_length: 565

code/configs/geneval_dpo_costa_1b_fp16.yml

@@ -0,0 +1,13 @@
+model_name: mnoukhov/dpo2_costa_1b_20k_fp16.yml_dff3275532270a8cbadb56d184c5d31d
+wandb_log_id: model_name
+base_model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+base_model_revision: sft__55513__1706646024
+gen_dtype: bfloat16
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 128
+max_length: 640
+temperature: 0.010001

code/configs/geneval_test.yml

@@ -0,0 +1,18 @@
+# model_revisions: ['sft__55513__1708611267']
+model_name: mnoukhov/dpo2_costa_1b_20k_fp16.yml_91ead4b5862c14e701bb164c36d54628
+model_revisions: ['step1']
+base_model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+base_model_revision: sft__55513__1706646024
+wandb_log_id: model_name
+# model_name: mnoukhov/EleutherAI_pythia-1b-deduped__sft__tldr_dpo_20konly_1b_fp16.yml_24e9f83_merged
+# model_revisions: ['step1']
+# wandb_log_id: 06936e8694635c9d13ec2d47abdeb0aa
+split: validation[:10]
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gen_dtype: bfloat16
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 128
+max_length: 565

code/configs/gptrm1b_eval_generated_pythia1b_fp16.yml

@@ -0,0 +1,11 @@
+dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_410m_dpo1
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+just_eval: True
+use_lora: False
+train_split: train
+eval_split: train
+load_in_8bit: False
+fp16_model: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8

code/configs/gptrm1b_eval_pythia1b_fp16.yml

@@ -0,0 +1,11 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+just_eval: True
+use_lora: False
+train_split: train
+eval_split: test
+load_in_8bit: False
+fp16_model: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8

code/configs/gptrm1b_evaltrain_pythia1b_fp16.yml

@@ -0,0 +1,11 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+just_eval: True
+use_lora: False
+train_split: train
+eval_split: train
+load_in_8bit: False
+fp16_model: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8

code/configs/gptrm1b_from_sft_pythia410m.yml

@@ -0,0 +1,18 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 4
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: adamw_torch
+num_train_epochs: 1
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+padding: do_not_pad
+eval_steps: 0.25

code/configs/gptrm1b_pythia160m_r8_alpha32.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia1b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia160m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 16
+per_device_eval_batch_size: 16
+gradient_accumulation_steps: 2
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: adamw_torch
+num_train_epochs: 3
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32

code/configs/gptrm1b_pythia410m_fp16_r8_alpha32_4V100.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia1b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp16_trainall_3epochs
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 4
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: adamw_torch
+num_train_epochs: 1
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32

code/configs/gptrm1b_pythia410m_fp16_r8_alpha32_4V100_nopad.yml

@@ -0,0 +1,18 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 4
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: adamw_torch
+num_train_epochs: 1
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+padding: do_not_pad
+eval_steps: 0.25

code/configs/gptrm1b_pythia410m_r8_alpha32_4V100.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia1b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32_trainall_3epochs
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 4
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: adamw_torch
+num_train_epochs: 3
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32

code/configs/gptrm_eval_adapter.yml

@@ -0,0 +1,11 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia7b_4V100_seq550
+pretrained_adapter: /home/toolkit/huggingface/tldr_gptrm_sft_pythia7b_4V100_seq560_fp16_3epochs_loralinear_adapter
+train_split: train[:1]
+eval_split: test[:5000]
+just_eval: True
+use_lora: False
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 4
+seq_length: 560

code/configs/gptrm_eval_pythia1b_fp16.yml

@@ -0,0 +1,10 @@
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+just_eval: True
+use_lora: False
+train_split: train[:1]
+eval_split: test[:5000]
+load_in_8bit: False
+fp16_model: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8

code/configs/gptrm_eval_sft_pythia7b_seq560_fp16_3epochs_loralinear.yml

@@ -0,0 +1,10 @@
+model_name: /home/toolkit/huggingface/tldr_gptrm_sft_pythia7b_4V100_seq560_fp16_3epochs_loralinear
+train_split: train[:1]
+eval_split: test[:5000]
+just_eval: True
+use_lora: False
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 4
+seq_length: 560

code/configs/gptrm_eval_sft_pythia7b_seq560_fp16_3epochs_loralinear_int8.yml

@@ -0,0 +1,10 @@
+model_name: /home/toolkit/huggingface/tldr_gptrm_sft_pythia7b_4V100_seq560_fp16_3epochs_loralinear
+just_eval: True
+use_lora: False
+train_split: train[:1]
+eval_split: test[:5000]
+load_in_8bit: True
+bf16: False
+fp16: False
+per_device_eval_batch_size: 4
+seq_length: 560

code/configs/gptrm_label_generated_pythia1b.yml

@@ -0,0 +1,12 @@
+mode: relabel
+output_dir: /home/toolkit/huggingface/openai_summarize_generated_20k_relabel_1b_margin
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+dataset_name: mnoukhov/openai_summarize_generated_20k
+train_split: train
+eval_split: null
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_eval_batch_size: 8
+padding: do_not_pad

code/configs/gptrm_label_pythia1b.yml

@@ -0,0 +1,12 @@
+mode: relabel
+output_dir: /home/toolkit/huggingface/openai_summarize_comparisons_tldrprompt_relabel1b_margin
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt
+train_split: train
+eval_split: test[:5000]
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_eval_batch_size: 8
+padding: do_not_pad

code/configs/gptrm_predict_generated_pythia1b.yml

@@ -0,0 +1,12 @@
+mode: predict
+output_dir: /home/toolkit/huggingface/openai_summarize_generated_20k_relabel_1b_margin
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+dataset_name: mnoukhov/openai_summarize_generated_20k
+train_split: train
+eval_split: null
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_eval_batch_size: 8
+padding: do_not_pad

code/configs/gptrm_pythia1b_trainall_fp16_4V100.yml

@@ -0,0 +1,15 @@
+model_name: EleutherAI/pythia-1b-deduped
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 8
+gradient_checkpointing: False
+learning_rate: 1e-6
+optimizer_type: adamw_torch
+lr_scheduler_type: linear
+num_train_epochs: 3
+seq_length: 560
+eval_steps: 250

code/configs/gptrm_sft_pythia14m_fp32.yml

@@ -0,0 +1,15 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia410m
+model_name: /home/toolkit/huggingface/tldr_sft_pythia14m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 128
+per_device_eval_batch_size: 32
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 1
+seq_length: 560
+eval_steps: 0.25

code/configs/gptrm_sft_pythia14m_fp32_2epochs.yml

@@ -0,0 +1,15 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia410m
+model_name: /home/toolkit/huggingface/tldr_sft_pythia14m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 128
+per_device_eval_batch_size: 32
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 2
+seq_length: 560
+eval_steps: 0.2

code/configs/gptrm_sft_pythia14m_fp32_3epochs.yml

@@ -0,0 +1,15 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia410m
+model_name: /home/toolkit/huggingface/tldr_sft_pythia14m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 128
+per_device_eval_batch_size: 32
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+eval_steps: 0.1

code/configs/gptrm_sft_pythia14m_fp32_3epochs_lr1e-6.yml

@@ -0,0 +1,15 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia410m
+model_name: /home/toolkit/huggingface/tldr_sft_pythia14m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 128
+per_device_eval_batch_size: 32
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-6
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+eval_steps: 0.1

code/configs/gptrm_sft_pythia1b_fp16_lora_4V100.yml

@@ -0,0 +1,16 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia1b_fp16_trainall_3epochs
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 4
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: adamw_torch
+num_train_epochs: 3
+seq_length: 560
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32

code/configs/gptrm_sft_pythia1b_trainall_fp16_4V100.yml

@@ -0,0 +1,16 @@
+model_name: mnoukhov/pythia1b-tldr-sft
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+gradient_accumulation_steps: 8
+gradient_checkpointing: False
+learning_rate: 1e-6
+optimizer_type: adamw_torch
+lr_scheduler_type: linear
+num_train_epochs: 2
+seq_length: 560
+eval_steps: 250
+padding: do_not_pad

code/configs/gptrm_sft_pythia1b_traincomp_fp16_4V100.yml

@@ -0,0 +1,15 @@
+model_name: /home/toolkit/trl/results/d31ad1f5a2ea94087cf49d8046228e62/code/results/checkpoint-1000
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 8
+gradient_checkpointing: False
+learning_rate: 1e-6
+optimizer_type: adamw_torch
+lr_scheduler_type: linear
+num_train_epochs: 3
+seq_length: 560
+eval_steps: 250

code/configs/gptrm_sft_pythia410m3_fp32_4V100.yml

@@ -0,0 +1,13 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32_trainall_3epochs
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 2
+gradient_checkpointing: False
+learning_rate: 5e-6
+optimizer_type: adamw_torch
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_amp16.yml

@@ -0,0 +1,13 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_amp16_batch128.yml

@@ -0,0 +1,14 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_amp16_batch8.yml

@@ -0,0 +1,14 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_fp16_loralinear_r32.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+use_lora: True
+lora_all_linear: True
+lora_r: 32

code/configs/gptrm_sft_pythia410m_fp16_loralinear_r8.yml

@@ -0,0 +1,17 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+use_lora: True
+lora_dropout: 0.01
+lora_all_linear: True
+lora_r: 8

code/configs/gptrm_sft_pythia410m_fp16_loralinear_r8_alpha32_batch128.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32

code/configs/gptrm_sft_pythia410m_fp32.yml

@@ -0,0 +1,14 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_fp32_adamtorch.yml

@@ -0,0 +1,14 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: adamw_torch
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_fp32_batch8.yml

@@ -0,0 +1,14 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_fp32_loralinear_r32_a16_batch8.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+lora_all_linear: True
+lora_r: 32
+lora_alpha: 16

code/configs/gptrm_sft_pythia410m_fp32_loralinear_r8_alpha32_batch128.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32

code/configs/gptrm_sft_pythia7b_4V100_seq550_fp16.yml

@@ -0,0 +1,11 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia7b_4V100_seq550
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 2
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 2

code/configs/gptrm_sft_pythia7b_4V100_seq560_fp16_3epochs.yml

@@ -0,0 +1,12 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia7b_4V100_seq550
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 2
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 5
+seq_length: 560

code/configs/gptrm_sft_pythia7b_4V100_seq560_fp16_3epochs_loralinear.yml

@@ -0,0 +1,13 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia7b_4V100_seq550
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 1
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 32
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+lora_all_linear: True

code/configs/gptrm_sft_pythia7b_4V100_seq560_fp16_3epochs_loralinear_lr2e5_pagedadamw.yml

@@ -0,0 +1,13 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia7b_4V100_seq550
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 1
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 32
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+lora_all_linear: True

code/configs/gptrm_sft_pythia7b_4V100_seq640_fp16_3epochs.yml

@@ -0,0 +1,12 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia7b_4V100_seq550
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 1
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 32
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 640

code/configs/gptrm_test.yml

@@ -0,0 +1,20 @@
+train_split: train[:1000]
+eval_split: test[:1000]
+# model_name: EleutherAI/pythia-14m
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia1b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp16_trainall_3epochs
+load_in_8bit: True
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 4
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: adamw_torch
+num_train_epochs: 1
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+padding: do_not_pad

code/configs/newdpo2_pythia2.8b_tldr.yml

@@ -0,0 +1,29 @@
+## model and dataset
+model_name: mnoukhov/EleutherAI_pythia-2.8b-deduped__sft__tldr_55513 
+# hub_model_id: "mnoukhov/pythia-2.8b-dpo_hh_rlhf"
+dataset_name: mnoukhov/summarize_from_feedback_tldr3_unlabelled_vllm_dpo_costa_2.8b_bf16.yml_6e799_new
+eval_dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+dataset_eval_split: validation
+report_to: "wandb"
+## dpo
+learning_rate: 1e-5
+lr_scheduler_type: cosine
+fp16: False
+bf16: True
+gradient_accumulation_steps: 8
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 4
+num_train_epochs: 1
+max_length: 640
+max_prompt_length: 512 
+max_target_length: 128
+beta: 0.05
+## peft
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+gradient_checkpointing: False
+evaluation_strategy: "steps"
+eval_steps: 0.2
+logging_steps: 100
+ddp_find_unused_parameters: False

code/configs/newdpo_pythia2.8b_hh.yml

@@ -0,0 +1,28 @@
+## model and dataset
+model_name: "mnoukhov/pythia-2.8b-sft_hh_rlhf"
+# output name
+hub_model_id: "mnoukhov/pythia-2.8b-dpo_hh_rlhf"
+dataset_name: "sophiex/hh-rlhf"
+report_to: "wandb"
+## dpo
+learning_rate: 3e-5
+lr_scheduler_type: cosine
+fp16: False
+bf16: True
+gradient_accumulation_steps: 8
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 4
+num_train_epochs: 1
+max_length: 512
+max_prompt_length: 256 
+max_target_length: 256
+beta: 0.1
+## peft
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+gradient_checkpointing: False
+evaluation_strategy: "steps"
+eval_steps: 0.2
+logging_steps: 100
+ddp_find_unused_parameters: False

code/configs/newdpo_pythia2.8b_hh_mitchell.yml

@@ -0,0 +1,29 @@
+## model and dataset
+model_name: "mnoukhov/pythia-2.8b-mitchell-sft_hh_rlhf"
+# output name
+hub_model_id: "mnoukhov/pythia-2.8b-mitchell-dpo_hh_rlhf"
+dataset_name: "sophiex/hh-rlhf"
+report_to: "wandb"
+## dpo
+learning_rate: 0.0000005
+lr_scheduler_type: constant_with_warmup
+warmup_steps: 150
+fp16: False
+bf16: True
+gradient_accumulation_steps: 16
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+num_train_epochs: 1
+max_length: 512
+max_prompt_length: 256 
+max_target_length: 256
+beta: 0.1
+## peft
+use_peft: False
+# lora_r: 16
+# lora_alpha: 32
+gradient_checkpointing: True
+evaluation_strategy: "steps"
+eval_steps: 0.2
+logging_steps: 100
+ddp_find_unused_parameters: False

code/configs/newsft_pythia2.8b_hh.yml

@@ -0,0 +1,23 @@
+model_name: "EleutherAI/pythia-2.8b-deduped"
+output_model_name: "mnoukhov/pythia-2.8b-sft_hh_rlhf"
+dataset_name: "sophiex/hh-rlhf"
+dataset_text_field: null
+report_to: "wandb"
+learning_rate: 1e-5
+lr_scheduler_type: cosine
+fp16: False
+bf16: True
+gradient_accumulation_steps: 32
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+num_train_epochs: 1
+max_seq_length: 1024
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+gradient_checkpointing: True
+evaluation_strategy: "steps"
+eval_steps: 0.2
+logging_steps: 100
+# ddp_find_unused_parameters: False
+sanity_check: True

code/configs/newsft_pythia410m_tldr.yml

@@ -0,0 +1,20 @@
+task_type: tldr
+model_name: EleutherAI/pythia-410m-deduped
+hub_model_id: mnoukhov/pythia410m-sft-tldr
+push_to_hub: True
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+dataset_eval_split: validation
+report_to: "wandb"
+learning_rate: 3e-6
+lr_scheduler_type: cosine
+bf16: True
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 32
+per_device_eval_batch_size: 8
+num_train_epochs: 1
+max_seq_length: 580
+gradient_checkpointing: False
+evaluation_strategy: "steps"
+eval_steps: 0.2
+logging_steps: 100
+ddp_find_unused_parameters: False

code/configs/newsft_test.yml

@@ -0,0 +1,22 @@
+output_dir: test
+model_name_or_path: "EleutherAI/pythia-410m"
+dataset_name: "CarperAI/openai_summarize_tldr"
+dataset_train_name: train[:100]
+dataset_test_name: valid[:100]
+report_to: "wandb"
+learning_rate: 1e-5
+lr_scheduler_type: cosine
+warmup_steps: 100
+weight_decay: 0.05
+fp16: True
+bf16: False
+gradient_checkpointing: False
+gradient_accumulation_steps: 1
+per_device_train_batch_size: 2
+per_device_eval_batch_size: 8
+num_train_epochs: 1
+max_seq_length: 560
+use_peft: True
+lora_r: 8
+lora_alpha: 32
+logging_steps: 10

code/configs/pseudo_pythia1b.yml

@@ -0,0 +1,9 @@
+output_dir: /home/toolkit/huggingface/openai_summarize_comparisons_tldrprompt_relabel1b
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt
+train_split: train
+eval_split: test[:5000]
+load_in_8bit: False
+fp16: True
+batch_size: 16
+seq_length: 560