初始化项目，由ModelHub XC社区提供模型

Model: mnoukhov/pythia410m-sft-tldr
Source: Original Platform

code/README.md

@@ -0,0 +1,4 @@
+# how to generate and psuedo label
+
+- generate with `generate_vllm.py`
+- pseudolabel with either `dpo_training.py` or `gpt_reward_modeling.py` by setting `mode = relabel`

code/callbacks.py

@@ -0,0 +1,471 @@
+import math
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import accelerate
+import torch
+from datasets import Dataset
+from torch.utils.data import DataLoader
+from tqdm.auto import tqdm
+from transformers import PreTrainedTokenizerBase, TrainerCallback
+
+import wandb
+from trl.trainer.utils import pad_to_length
+
+
+@dataclass
+class PromptAndTextCollator:
+    tokenizer: PreTrainedTokenizerBase
+    padding: Union[bool, str] = True
+    max_prompt_length: Optional[int] = None
+    max_length: Optional[int] = None
+    prompt_field: str = "prompt"
+    target_field: str = "label"
+    return_tensors: str = "pt"
+
+    def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, Any]:
+        prompts = [feat[self.prompt_field] for feat in features]
+        texts = [feat[self.prompt_field] + " " + feat[self.target_field] for feat in features]
+
+        original_side = self.tokenizer.padding_side
+        self.tokenizer.padding_side = "left"
+
+        tokenized_batch = self.tokenizer(
+            prompts,
+            truncation=True,
+            padding=True,
+            max_length=self.max_prompt_length,
+            return_tensors=self.return_tensors,
+        )
+        tokenized_batch["prompt"] = prompts
+
+        self.tokenizer.padding_side = original_side
+
+        tokenized_texts = self.tokenizer(
+            texts,
+            truncation=True,
+            padding=True,
+            max_length=self.max_length,
+            return_tensors=self.return_tensors,
+        )
+
+        text_labels = tokenized_texts["input_ids"].clone()
+        if self.tokenizer.pad_token_id is not None:
+            text_labels[text_labels == self.tokenizer.pad_token_id] = -100
+
+        tokenized_batch.update(
+            {
+                "text_input_ids": tokenized_texts["input_ids"],
+                "text_attention_mask": tokenized_texts["attention_mask"],
+                "text_labels": text_labels,
+            }
+        )
+
+        return tokenized_batch
+
+
+class GoldModelRewardCallback(TrainerCallback):
+    def __init__(
+        self,
+        args,
+        gold_model,
+        gold_eval_dataset,
+        tokenizer,
+        accelerator,
+        max_length,
+        max_prompt_length,
+        prompt_field,
+        target_field,
+        gold_load_and_unload=False,
+        log_n_samples_during_eval=0,
+        generation_config=None,
+    ):
+        self.max_length = max_length
+        self.log_n_samples_during_eval = log_n_samples_during_eval
+        self.generation_config = generation_config
+
+        # data_collator = DataCollatorWithPadding(tokenizer)
+        data_collator = PromptAndTextCollator(
+            tokenizer,
+            max_prompt_length=max_prompt_length,
+            max_length=max_length,
+            prompt_field=prompt_field,
+            target_field=target_field,
+        )
+        dataloader_params = {
+            "batch_size": args.eval_batch_size,
+            "collate_fn": data_collator,
+            "num_workers": args.dataloader_num_workers,
+            "pin_memory": args.dataloader_pin_memory,
+        }
+        dataloader = DataLoader(gold_eval_dataset, **dataloader_params)
+        self.dataloader = accelerator.prepare(dataloader)
+        self.accelerator = accelerator
+        self.completed_step = -1
+        self.gold_model = gold_model
+        self.gold_load_and_unload = gold_load_and_unload
+        # keep model on gpu the whole time
+        if not self.gold_load_and_unload:
+            self.gold_model = self.accelerator.prepare(self.gold_model)
+
+    def on_evaluate(self, args, state, control, model, tokenizer, metrics, **kwargs):
+        samples_to_log = []
+        gold_reward_sum = 0.0
+        nll_sum = 0.0
+        total_samples = 0
+        sample_length_sum = 0.0
+
+        # load model onto gpu for inference then unload
+        if self.gold_load_and_unload:
+            self.gold_model = self.accelerator.prepare(self.gold_model)
+
+        if state.global_step == self.completed_step:
+            return
+
+        for inputs in tqdm(
+            self.dataloader, desc="Gold Eval", dynamic_ncols=True, disable=not state.is_local_process_zero
+        ):
+            # get loss over true continuation i.e. ppl on dataset
+            with torch.no_grad():
+                nll_loss = model(
+                    input_ids=inputs["text_input_ids"],
+                    attention_mask=inputs["text_attention_mask"],
+                    labels=inputs["text_labels"],
+                ).loss
+
+            nll_loss = self.accelerator.gather_for_metrics(nll_loss)
+
+            # generate from model
+            policy_output_decoded, ref_output_decoded, policy_output_ids = self.get_batch_samples(
+                model,
+                tokenizer,
+                inputs["input_ids"],
+                inputs["attention_mask"],
+                return_ids=True,
+            )
+
+            # gold reward
+            policy_output_attention_mask = (policy_output_ids != tokenizer.pad_token_id).to(torch.int64)
+            with torch.no_grad():
+                gold_rewards = self.gold_model(
+                    input_ids=policy_output_ids, attention_mask=policy_output_attention_mask
+                )[0]
+
+            gold_rewards = self.accelerator.gather_for_metrics(gold_rewards)
+
+            if state.is_local_process_zero:
+                nll_sum += nll_loss.sum().item()
+                gold_reward_sum += gold_rewards.sum().item()
+                total_samples += gold_rewards.size(0)
+                sample_length_sum += policy_output_attention_mask.sum().item()
+
+                # Sample and save to game log if requested (for one batch to save time)
+                for i, (prompt, pol, ref) in enumerate(
+                    zip(inputs["prompt"], policy_output_decoded, ref_output_decoded)
+                ):
+                    if len(samples_to_log) < self.log_n_samples_during_eval:
+                        samples_to_log.append([prompt, pol[len(prompt) :], ref[len(prompt) :]])
+                    else:
+                        break
+
+        if self.gold_load_and_unload:
+            self.gold_model = self.gold_model.to("cpu")
+            torch.cuda.empty_cache()
+
+        if state.is_world_process_zero:
+            gold_log = {
+                "eval/gold_rewards_mean": gold_reward_sum / total_samples,
+                "eval/perplexity": math.exp(nll_sum / total_samples),
+                "eval/gold_sample_length": sample_length_sum / total_samples,
+            }
+            for key, value in gold_log.items():
+                print(f"{key}: {value}")
+            if state.epoch:
+                gold_log["epoch"] = round(state.epoch, 2)
+                gold_log["step"] = state.global_step
+            if samples_to_log:
+                gold_log["gold_log"] = (
+                    wandb.Table(
+                        columns=["Prompt", "Policy", "Ref Model"],
+                        rows=samples_to_log,
+                    ),
+                )
+            wandb.log(gold_log)
+
+        self.completed_step = state.global_step
+
+    def get_batch_samples(self, model, tokenizer, input_ids, attention_mask, return_ids=False) -> Tuple[str, str]:
+        """Reduce inputs to unseen prompts, and maximum batch size if necessary
+        Generate samples from the model and reference model for the given batch of inputs."""
+        policy_output = model.generate(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            generation_config=self.generation_config,
+        )
+
+        # if self.ref_model is None:
+        with self.accelerator.unwrap_model(model).disable_adapter():
+            reference_output = model.generate(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                generation_config=self.generation_config,
+            )
+        # else:
+        #     reference_output = self.ref_model.generate(
+        #         **inputs,
+        #         generation_config=self.generation_config,
+        #     )
+
+        policy_output = pad_to_length(policy_output, self.max_length, tokenizer.pad_token_id)
+        policy_output_decoded = tokenizer.batch_decode(policy_output, skip_special_tokens=True)
+
+        reference_output = pad_to_length(reference_output, self.max_length, tokenizer.pad_token_id)
+        reference_output_decoded = tokenizer.batch_decode(reference_output, skip_special_tokens=True)
+
+        if return_ids:
+            return policy_output_decoded, reference_output_decoded, policy_output
+        else:
+            return policy_output_decoded, reference_output_decoded
+
+
+class PerplexityCallback(TrainerCallback):
+    """Like GoldModelReward in that you generate and get ppl on dataset
+
+    But you don't run eval with the gold model
+    Useful when gold model is very larger and you want to run inference later
+    """
+
+    def __init__(
+        self,
+        args,
+        dataset,
+        tokenizer,
+        accelerator,
+        max_length,
+        max_prompt_length,
+        prompt_field,
+        target_field,
+        hub_model_id=None,
+        **kwargs,
+    ):
+        self.max_length = max_length
+
+        # data_collator = DataCollatorWithPadding(tokenizer)
+        data_collator = PromptAndTextCollator(
+            tokenizer,
+            max_prompt_length=max_prompt_length,
+            max_length=max_length,
+            prompt_field=prompt_field,
+            target_field=target_field,
+        )
+        dataloader_params = {
+            "batch_size": args.eval_batch_size,
+            "collate_fn": data_collator,
+            "num_workers": args.dataloader_num_workers,
+            "pin_memory": args.dataloader_pin_memory,
+        }
+        dataloader = DataLoader(dataset, **dataloader_params)
+        self.dataloader = accelerator.prepare(dataloader)
+        self.accelerator = accelerator
+        self.completed_step = -1
+        self.hub_model_id = hub_model_id
+
+    def on_evaluate(self, args, state, control, model, tokenizer, metrics, **kwargs):
+        nll_sum = 0.0
+        total_samples = 0
+
+        if state.global_step == self.completed_step:
+            return
+
+        for inputs in tqdm(
+            self.dataloader, desc="PPL and Gen Eval", dynamic_ncols=True, disable=not state.is_local_process_zero
+        ):
+            # get loss over true continuation i.e. ppl on dataset
+            with torch.no_grad():
+                nll_loss = model(
+                    input_ids=inputs["text_input_ids"],
+                    attention_mask=inputs["text_attention_mask"],
+                    labels=inputs["text_labels"],
+                ).loss
+
+            nll_loss = self.accelerator.gather_for_metrics(nll_loss)
+
+            if state.is_local_process_zero:
+                total_samples += nll_loss.size(0)
+                nll_sum += nll_loss.sum().item()
+
+        if state.is_world_process_zero:
+            # gather_for_metrics doesn't work for list of strings?
+            gold_log = {
+                "eval/perplexity": math.exp(nll_sum / total_samples),
+            }
+            for key, value in gold_log.items():
+                print(f"{key}: {value}")
+            if state.epoch:
+                gold_log["epoch"] = round(state.epoch, 2)
+                gold_log["step"] = state.global_step
+
+            wandb.log(gold_log)
+
+            if self.hub_model_id is not None:
+                model.push_to_hub(self.hub_model_id, revision=f"step{state.global_step}")
+
+        self.completed_step = state.global_step
+
+
+class PerplexityGenCallback(TrainerCallback):
+    """Like GoldModelReward in that you generate and get ppl on dataset
+
+    But you don't run eval with the gold model
+    Useful when gold model is very larger and you want to run inference later
+    """
+
+    def __init__(
+        self,
+        args,
+        dataset,
+        tokenizer,
+        accelerator,
+        max_length,
+        max_prompt_length,
+        prompt_field,
+        target_field,
+        log_n_samples_during_eval=0,
+        generation_config=None,
+        hub_model_id="tmp",
+    ):
+        self.max_length = max_length
+        self.log_n_samples_during_eval = log_n_samples_during_eval
+        self.generation_config = generation_config
+
+        # data_collator = DataCollatorWithPadding(tokenizer)
+        data_collator = PromptAndTextCollator(
+            tokenizer,
+            max_prompt_length=max_prompt_length,
+            max_length=max_length,
+            prompt_field=prompt_field,
+            target_field=target_field,
+        )
+        dataloader_params = {
+            "batch_size": args.eval_batch_size,
+            "collate_fn": data_collator,
+            "num_workers": args.dataloader_num_workers,
+            "pin_memory": args.dataloader_pin_memory,
+        }
+        dataloader = DataLoader(dataset, **dataloader_params)
+        self.dataloader = accelerator.prepare(dataloader)
+        self.accelerator = accelerator
+        self.completed_step = -1
+        self.hub_name = hub_model_id
+
+    def on_evaluate(self, args, state, control, model, tokenizer, metrics, **kwargs):
+        all_generations = []
+        all_prompts = []
+        nll_sum = 0.0
+        total_samples = 0
+        sample_length_sum = 0.0
+
+        if state.global_step == self.completed_step:
+            return
+
+        for inputs in tqdm(
+            self.dataloader, desc="PPL and Gen Eval", dynamic_ncols=True, disable=not state.is_local_process_zero
+        ):
+            # get loss over true continuation i.e. ppl on dataset
+            with torch.no_grad():
+                nll_loss = model(
+                    input_ids=inputs["text_input_ids"],
+                    attention_mask=inputs["text_attention_mask"],
+                    labels=inputs["text_labels"],
+                ).loss
+
+            # generate from model
+            policy_output_ids = model.generate(
+                input_ids=inputs["input_ids"],
+                attention_mask=inputs["attention_mask"],
+                generation_config=self.generation_config,
+            )
+            policy_output_ids = pad_to_length(policy_output_ids, self.max_length, tokenizer.pad_token_id)
+
+            policy_output_attention_mask = (policy_output_ids != tokenizer.pad_token_id).to(torch.int64)
+            generation_sizes = policy_output_attention_mask.sum(dim=1)
+
+            (nll_loss, generation_ids, generation_sizes) = self.accelerator.gather_for_metrics(
+                (nll_loss, policy_output_ids, generation_sizes)
+            )
+
+            prompts = accelerate.utils.gather_object(inputs["prompt"])
+
+            if state.is_local_process_zero:
+                nll_sum += nll_loss.sum().item()
+                total_samples += generation_sizes.size(0)
+                sample_length_sum += generation_sizes.sum().item()
+                generation_strs = tokenizer.batch_decode(generation_ids, skip_special_tokens=True)
+                all_prompts.extend(prompts)
+                all_generations.extend(generation_strs)
+
+        if state.is_world_process_zero:
+            # gather_for_metrics doesn't work for list of strings?
+            gold_log = {
+                "eval/perplexity": math.exp(nll_sum / total_samples),
+                "eval/gold_sample_length": sample_length_sum / total_samples,
+            }
+            for key, value in gold_log.items():
+                print(f"{key}: {value}")
+            if state.epoch:
+                gold_log["epoch"] = round(state.epoch, 2)
+                gold_log["step"] = state.global_step
+
+            if self.log_n_samples_during_eval:
+                samples_to_log = [
+                    [prompt, generation[len(prompt) :]]
+                    for prompt, generation in zip(
+                        all_prompts[: self.log_n_samples_during_eval],
+                        all_generations[: self.log_n_samples_during_eval],
+                    )
+                ]
+                gold_log["gold_log"] = (
+                    wandb.Table(
+                        columns=["Prompt", "Policy"],
+                        rows=samples_to_log,
+                    ),
+                )
+
+            wandb.log(gold_log)
+            generation_ds = Dataset.from_dict({"generations": all_generations})
+            generation_ds.push_to_hub(f"{self.hub_name}_generations", revision=str(state.global_step))
+
+        self.completed_step = state.global_step
+
+    def get_batch_samples(self, model, tokenizer, input_ids, attention_mask, return_ids=False) -> Tuple[str, str]:
+        """Reduce inputs to unseen prompts, and maximum batch size if necessary
+        Generate samples from the model and reference model for the given batch of inputs."""
+        policy_output = model.generate(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            generation_config=self.generation_config,
+        )
+
+        # if self.ref_model is None:
+        with self.accelerator.unwrap_model(model).disable_adapter():
+            reference_output = model.generate(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                generation_config=self.generation_config,
+            )
+        # else:
+        #     reference_output = self.ref_model.generate(
+        #         **inputs,
+        #         generation_config=self.generation_config,
+        #     )
+
+        policy_output = pad_to_length(policy_output, self.max_length, tokenizer.pad_token_id)
+        policy_output_decoded = tokenizer.batch_decode(policy_output, skip_special_tokens=True)
+
+        reference_output = pad_to_length(reference_output, self.max_length, tokenizer.pad_token_id)
+        reference_output_decoded = tokenizer.batch_decode(reference_output, skip_special_tokens=True)
+
+        if return_ids:
+            return policy_output_decoded, reference_output_decoded, policy_output
+        else:
+            return policy_output_decoded, reference_output_decoded

code/configs/accelerate_zero2_4gpu.yml

@@ -0,0 +1,20 @@
+compute_environment: LOCAL_MACHINE
+debug: false
+deepspeed_config:
+  offload_optimizer_device: none
+  offload_param_device: none
+  zero3_init_flag: false
+  zero_stage: 2
+distributed_type: DEEPSPEED
+downcast_bf16: 'no'
+machine_rank: 0
+main_training_function: main
+mixed_precision: 'no'
+num_machines: 1
+num_processes: 4
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

code/configs/create_rlhf_410m.yml

@@ -0,0 +1,11 @@
+output_dir: /home/toolkit/huggingface/openai_summarize_tldr_rbaseline
+train_split: train
+eval_split: valid[:2000]
+###
+model_name: mnoukhov/pythia410m-tldr-sft-rm-adapter
+new_column_name: reward_baseline
+dataset_name: CarperAI/openai_summarize_tldr
+load_in_8bit: False
+fp16: True
+batch_size: 32
+max_length: 560

code/configs/create_rlhf_410m_1b.yml

@@ -0,0 +1,11 @@
+output_dir: /home/toolkit/huggingface/openai_summarize_tldr_grbaseline
+train_split: train
+eval_split: valid[:2000]
+###
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+new_column_name: gold_reward_baseline
+dataset_name: mnoukhov/openai_summarize_tldr_rbaseline
+load_in_8bit: False
+fp16: True
+batch_size: 32
+max_length: 560

code/configs/dpo1b2_10k_pythia410m_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_10k
+beta: 0.5
+num_train_epochs: 5
+eval_steps: 750
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20k-reuse_pythia410m_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_comparisons_20k_regen_and_relabelled
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20k_pythia410m-iter1_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_410m_dpo1
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20k_pythia410m_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_410m_dpo1
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20kgold_pythia410m-iter1_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_1b
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20kgold_pythia410m_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_1b
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20kgoldonly_pythia410m-iter1_fp16.yml

@@ -0,0 +1,20 @@
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+train_split: train[:1]
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_1b
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20kgoldonly_pythia410m_fp16.yml

@@ -0,0 +1,20 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+train_split: train[:1]
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_1b
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20konly-reuse_pythia410m_fp16.yml

@@ -0,0 +1,20 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+train_split: train[:1]
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_comparisons_20k_regen_and_relabelled
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20konly_pythia410m-iter1_fp16.yml

@@ -0,0 +1,20 @@
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+train_split: train[:1]
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_410m_dpo1
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_20konly_pythia410m_fp16.yml

@@ -0,0 +1,20 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+train_split: train[:1]
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_410m_dpo1
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b2_a100.yml

@@ -0,0 +1,20 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+train_split: train[:1]
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+pseudo_dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_410m_dpo1
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: True
+fp16: False
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 16
+warmup_steps: 150

code/configs/dpo1b_eval_generated_pythia410m_fp16.yml

@@ -0,0 +1,11 @@
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: mnoukhov/openai_comparisons_20k_regen_and_relabelled
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8
+warmup_steps: 150
+mode: eval

code/configs/dpo1b_eval_pythia410m_fp16.yml

@@ -0,0 +1,19 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+beta: 0.5
+num_train_epochs: 5
+eval_steps: 750
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150
+just_eval: True

code/configs/dpo1b_eval_regenerated_pythia410m_fp16.yml

@@ -0,0 +1,11 @@
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: arianhosseini/openai_comparisons_20k_regen_and_relabelled
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8
+warmup_steps: 150
+mode: eval

code/configs/dpo1b_predict_generated_pythia410m-dpo1.yml

@@ -0,0 +1,11 @@
+output_dir: /home/toolkit/huggingface/openai_summarize_generated_20k_relabel_1b_predict_410m-dpo1
+mode: predict
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_1b_margin
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8

code/configs/dpo1b_pythia410m_fp16.yml

@@ -0,0 +1,18 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo1b_relabel_comparisons.yml

@@ -0,0 +1,12 @@
+output_dir: /home/toolkit/huggingface/openai_summarize_comparisons_relabelled_margin
+mode: relabel
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8
+warmup_steps: 150

code/configs/dpo1b_relabel_generated_pythia410m_fp16.yml

@@ -0,0 +1,12 @@
+output_dir: /home/toolkit/huggingface/openai_summarize_generated_20k_relabelled_margin
+mode: relabel
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: mnoukhov/openai_summarize_generated_20k
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8
+warmup_steps: 150

code/configs/dpo1b_relabel_generated_same_prompts.yml

@@ -0,0 +1,12 @@
+output_dir: /home/toolkit/huggingface/openai_comparisons_20k_regen_and_relabelled
+mode: relabel
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: arianhosseini/openai_comparisons_20k_regen_and_relabelled
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8
+warmup_steps: 150

code/configs/dpo1b_relabel_vllm_generated_pythia410m.yml

@@ -0,0 +1,12 @@
+output_dir: openai_summarize_vllm_generated_20k_label410m
+mode: relabel
+model_name: mnoukhov/pythia410m-tldrprompt-dpo1b-adapter
+dataset_name: mnoukhov/openai_summarize_vllm_generated_20k
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8
+warmup_steps: 150

code/configs/dpo1b_test.yml

@@ -0,0 +1,19 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32_trainall_3epochs
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia1b
+beta: 0.5
+num_train_epochs: 3
+eval_steps: 750
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150
+eval_steps: 10
+save_steps: 10

code/configs/dpo1b_vllm_pythia410m.yml

@@ -0,0 +1,18 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_vllm_generated_20k_label410m
+gold_model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo2_costa_1b_20k_bf16.yml

@@ -0,0 +1,36 @@
+## dpo 2 
+pseudo_dataset_name: mnoukhov/summarize_from_feedback_tldr3_generated_20k_relabel_pythia1b_dpo_temp0.7_length128
+train_split: train[:1]
+max_prompt_length: 512
+max_target_length: 131
+max_length: 640
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.5
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: True
+fp16: False
+learning_rate: 3e-6
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 16
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo2_costa_1b_20k_fp16.yml

@@ -0,0 +1,35 @@
+## dpo 2 
+pseudo_dataset_name: mnoukhov/summarize_from_feedback_tldr3_generated_relabel_20k_dpo_costa_1b_fp16.yml_3d94f50_b9ff2
+train_split: train[:1]
+max_prompt_length: 512
+max_target_length: 131
+max_length: 640
+lr_scheduler_type: cosine
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.05
+max_steps: -1
+num_train_epochs: 2
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4

code/configs/dpo2_pythia2.8b_tldr.yml

@@ -0,0 +1,34 @@
+pseudo_dataset_name: mnoukhov/summarize_from_feedback_tldr3_unlabelled_vllm_dpo_costa_2.8b_bf16.yml_6e799_new
+train_split: train[:1]
+# dpo 2
+eval_first_step: False
+model_name: mnoukhov/EleutherAI_pythia-2.8b-deduped__sft__tldr_55513 
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_prompt_length: 512
+max_target_length: 131
+max_length: 640
+lr_scheduler_type: cosine
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.05
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: True
+fp16: False
+learning_rate: 1e-5
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.
+gradient_accumulation_steps: 16
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4

code/configs/dpo_1b_bf16.yml

@@ -0,0 +1,28 @@
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+eval_split: validation
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr
+gold_model_revision: reward__55513__1706651113
+gold_dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144 
+gold_prompt_field: query
+gold_eval_split: validation
+strip_prompt: False
+## training stuff
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: True
+fp16: False
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 16
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo_1b_fp16.yml

@@ -0,0 +1,31 @@
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_target_length: 128
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.5
+max_steps: -1
+num_train_epochs: 2
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo_20konly_1b_bf16.yml

@@ -0,0 +1,32 @@
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+eval_split: validation
+prompt_field: query
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr
+gold_model_revision: reward__55513__1706651113
+gold_dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144 
+gold_prompt_field: query
+gold_target_field: reference_response
+gold_eval_split: validation
+strip_prompt: False
+## training stuff
+eval_first_step: False
+pseudo_dataset_name: mnoukhov/summarize_from_feedback_tldr3_generated_20k_relabel_pythia1b_dpo
+beta: 0.5
+max_steps: 10000
+eval_steps: 1000
+load_in_8bit: False
+bf16: True
+fp16: False
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 16
+per_device_train_batch_size: 4
+warmup_steps: 150

code/configs/dpo_20konly_1b_fp16.yml

@@ -0,0 +1,33 @@
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+pseudo_dataset_name: mnoukhov/summarize_from_feedback_tldr3_generated_20k_relabel_pythia1b_dpo
+max_target_length: 128
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+train_split: train[:1]
+beta: 0.5
+max_steps: -1
+num_train_epochs: 5
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo_costa_1b_constantlr_fp16.yml

@@ -0,0 +1,32 @@
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_target_length: 169
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.5
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-6
+lr_scheduler_type: constant_with_warmup
+use_peft: True
+lora_all_linear: True
+lora_r: 32
+lora_alpha: 64
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo_costa_1b_fp16.yml

@@ -0,0 +1,32 @@
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_prompt_length: 512
+max_target_length: 131
+max_length: 640
+lr_scheduler_type: cosine
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+gold_eval: ppl
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.05
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4

code/configs/dpo_eval_1b_fp16.yml

@@ -0,0 +1,32 @@
+mode: eval
+push_to_hub: False
+gold_eval: none
+## costa stuff
+model_name: mnoukhov/EleutherAI_pythia-1b-deduped__sft__tldr_dpo_1b_fp16.yml_24e9f83_merged
+model_revision: step2324
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_target_length: 128
+## hub stuff
+push_to_hub_organization: mnoukhov
+## training stuff
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.5
+max_steps: -1
+num_train_epochs: 2
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo_eval_costa_1b_bf16.yml

@@ -0,0 +1,36 @@
+mode: eval
+push_to_hub: False
+gold_eval: none
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__dpo__tldr
+model_revision: dpo__55513__1707379566
+ref_model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+ref_model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_prompt_length: 512
+max_target_length: 169
+max_length: 638
+## hub stuff
+push_to_hub_organization: mnoukhov
+## training stuff
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.5
+max_steps: -1
+num_train_epochs: 2
+load_in_8bit: False
+bf16: True
+fp16: False
+learning_rate: 1e-5
+use_peft: False
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo_eval_costa_1b_fp16.yml

@@ -0,0 +1,34 @@
+mode: eval
+push_to_hub: False
+gold_eval: none
+## costa stuff
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__dpo__tldr
+model_revision: dpo__55513__1707379566
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: query
+eval_split: validation
+max_prompt_length: 512
+max_target_length: 169
+max_length: 638
+## hub stuff
+push_to_hub_organization: mnoukhov
+## training stuff
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.5
+max_steps: -1
+num_train_epochs: 2
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+warmup_steps: 150

code/configs/dpo_pythia1b_hh_rlhf.yml

@@ -0,0 +1,36 @@
+## costa stuff
+model_name: sophiex/pythia-1b-sft_hh_rlhf
+# model_revision: null
+dataset_name: sophiex/hh-rlhf
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: prompt
+eval_split: test
+max_prompt_length: 256
+max_target_length: 256
+max_length: 512
+lr_scheduler_type: cosine
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: sophiex
+## training stuff
+save_strategy: steps
+gold_eval: none
+gold_dataset_name: sophiex/hh-rlhf
+gold_target_field: chosen
+gold_eval_split: test
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.1
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4

code/configs/dpo_pythia1b_hh_rlhf_fp16_4V100.yml

@@ -0,0 +1,36 @@
+## costa stuff
+model_name: sophiex/pythia-1b-sft_hh_rlhf
+# model_revision: null
+dataset_name: sophiex/hh-rlhf
+tokenizer_name: EleutherAI/pythia-1b-deduped
+prompt_field: prompt
+eval_split: test
+max_prompt_length: 256
+max_target_length: 256
+max_length: 512
+lr_scheduler_type: cosine
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+save_strategy: steps
+gold_eval: ppl
+gold_dataset_name: sophiex/hh-rlhf
+gold_target_field: chosen
+gold_eval_split: test
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.1
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4

code/configs/dpo_pythia2.8b_hh_rlhf_fp16_4V100.yml

@@ -0,0 +1,36 @@
+## costa stuff
+model_name: sophiex/pythia-2.8b-sft_hh_rlhf
+# model_revision: null
+dataset_name: sophiex/hh-rlhf
+tokenizer_name: EleutherAI/pythia-2.8b-deduped
+prompt_field: prompt
+eval_split: test
+max_prompt_length: 256
+max_target_length: 256
+max_length: 512
+lr_scheduler_type: cosine
+## hub stuff
+push_to_hub: True
+push_to_hub_organization: mnoukhov
+## training stuff
+save_strategy: steps
+gold_eval: ppl
+gold_dataset_name: sophiex/hh-rlhf
+gold_target_field: chosen
+gold_eval_split: test
+eval_steps: 0.2
+save_steps: 0.2
+beta: 0.1
+max_steps: -1
+num_train_epochs: 1
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4

code/configs/dpo_relabel.yml

@@ -0,0 +1,19 @@
+output_dir: summarize_from_feedback_tldr3_generated_20k_relabel_pythia1b_dpo_temp0.7_length128
+mode: relabel
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__dpo__tldr
+model_revision: dpo__55513__1707379566 
+ref_model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+ref_model_revision: sft__55513__1706646024
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: mnoukhov/summarize_from_feedback_tldr3_generated_20k_vllm_pythia1b_dpo_temp0.7_length128
+max_prompt_length: 512
+max_target_length: 128
+max_length: 640
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: True
+fp16: False
+per_device_eval_batch_size: 8
+warmup_steps: 150

code/configs/dpo_relabel_summarize_generated_1b_dpo.yml

@@ -0,0 +1,19 @@
+output_dir: summarize_from_feedback_tldr3_generated_20k_relabel_pythia1b_dpo_temp0.7_length128
+mode: relabel
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__dpo__tldr
+model_revision: dpo__55513__1707379566 
+ref_model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+ref_model_revision: sft__55513__1706646024
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: mnoukhov/summarize_from_feedback_tldr3_generated_20k_vllm_pythia1b_dpo_temp0.7_length128
+max_prompt_length: 512
+max_target_length: 128
+max_length: 640
+eval_split: train
+use_peft: False
+beta: 0.5
+load_in_8bit: False
+bf16: True
+fp16: False
+per_device_eval_batch_size: 8
+warmup_steps: 150

code/configs/dpo_test.yml

@@ -0,0 +1,24 @@
+train_split: train[:1000]
+eval_split: validation[:10]
+##
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revision: sft__55513__1706646024
+dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+prompt_field: query
+gold_eval: ppl
+beta: 0.5
+num_train_epochs: 3
+eval_steps: 750
+load_in_8bit: False
+bf16: False
+fp16: True
+learning_rate: 1e-5
+use_peft: True
+lora_r: 8
+lora_alpha: 32
+lora_dropout: 0.05
+gradient_accumulation_steps: 4
+per_device_train_batch_size: 4
+warmup_steps: 150
+save_steps: 100
+eval_first_step: False

code/configs/geneval_costa_1b_dpo.yml

@@ -0,0 +1,12 @@
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__dpo__tldr
+model_revisions: ["dpo__55513__1707379566"]
+gen_dtype: bfloat16
+wandb_log_id: EleutherAI_pythia-1b-deduped__dpo__tldr_55513_length128
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 128
+max_length: 640
+temperature: 0.010001

code/configs/geneval_costa_1b_dpo_repro.yml

@@ -0,0 +1,12 @@
+model_name: mnoukhov/EleutherAI_pythia-1b-deduped__dpo__tldr
+model_revisions: ["dpo__55513__1712777528"]
+gen_dtype: bfloat16
+wandb_log_id: EleutherAI_pythia-1b-deduped__dpo__tldr_55513_length128_repro
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 128
+max_length: 630
+temperature: 0.010001

code/configs/geneval_costa_1b_ppo.yml

@@ -0,0 +1,12 @@
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__ppo_left_padding_new_nowhiten_reward__tldr
+model_revisions: ["ppo_left_padding_new_nowhiten_reward__55513__1709671967"]
+gen_dtype: bfloat16
+wandb_log_id: EleutherAI_pythia-1b-deduped__ppo_left_padding_new_nowhiten_reward__tldr_55513
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 53
+max_length: 565
+temperature: 0.010001

code/configs/geneval_costa_1b_sft.yml

@@ -0,0 +1,12 @@
+model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+model_revisions: ["sft__55513__1708611267"]
+gen_dtype: bfloat16
+wandb_log_id: EleutherAI_pythia-1b-deduped__sft__tldr_55513
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 128
+max_length: 640
+temperature: 0.010001

code/configs/geneval_dpo2_costa_1b_20k_fp16.yml

@@ -0,0 +1,11 @@
+model_name: mnoukhov/dpo2_pythia2.8b_tldr.yml_7692b3a0462f2e8fd35cc26b99936469
+wandb_log_id: model_name
+gen_dtype: bfloat16
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 128
+max_length: 640
+temperature: 0.010001

code/configs/geneval_dpo_20konly_1b_fp16.yml

@@ -0,0 +1,10 @@
+model_name: mnoukhov/EleutherAI_pythia-1b-deduped__sft__tldr_dpo_20konly_1b_fp16.yml_24e9f83_merged
+gen_dtype: bfloat16
+wandb_log_id: 06936e8694635c9d13ec2d47abdeb0aa
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 53
+max_length: 565

code/configs/geneval_dpo_costa_1b_fp16.yml

@@ -0,0 +1,13 @@
+model_name: mnoukhov/dpo2_costa_1b_20k_fp16.yml_dff3275532270a8cbadb56d184c5d31d
+wandb_log_id: model_name
+base_model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+base_model_revision: sft__55513__1706646024
+gen_dtype: bfloat16
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 128
+max_length: 640
+temperature: 0.010001

code/configs/geneval_test.yml

@@ -0,0 +1,18 @@
+# model_revisions: ['sft__55513__1708611267']
+model_name: mnoukhov/dpo2_costa_1b_20k_fp16.yml_91ead4b5862c14e701bb164c36d54628
+model_revisions: ['step1']
+base_model_name: vwxyzjn/EleutherAI_pythia-1b-deduped__sft__tldr
+base_model_revision: sft__55513__1706646024
+wandb_log_id: model_name
+# model_name: mnoukhov/EleutherAI_pythia-1b-deduped__sft__tldr_dpo_20konly_1b_fp16.yml_24e9f83_merged
+# model_revisions: ['step1']
+# wandb_log_id: 06936e8694635c9d13ec2d47abdeb0aa
+split: validation[:10]
+tokenizer_name: EleutherAI/pythia-1b-deduped
+dataset_name: vwxyzjn/summarize_from_feedback_tldr_3_filtered_oai_preprocessing_1706381144
+gen_dtype: bfloat16
+gold_model_name: vwxyzjn/EleutherAI_pythia-6.9b-deduped__reward__tldr 
+gold_model_revision: reward__55513__1708628552
+eval_dtype: bfloat16
+max_new_tokens: 128
+max_length: 565

code/configs/gptrm1b_eval_generated_pythia1b_fp16.yml

@@ -0,0 +1,11 @@
+dataset_name: mnoukhov/openai_summarize_generated_20k_relabel_410m_dpo1
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+just_eval: True
+use_lora: False
+train_split: train
+eval_split: train
+load_in_8bit: False
+fp16_model: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8

code/configs/gptrm1b_eval_pythia1b_fp16.yml

@@ -0,0 +1,11 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+just_eval: True
+use_lora: False
+train_split: train
+eval_split: test
+load_in_8bit: False
+fp16_model: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8

code/configs/gptrm1b_evaltrain_pythia1b_fp16.yml

@@ -0,0 +1,11 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+just_eval: True
+use_lora: False
+train_split: train
+eval_split: train
+load_in_8bit: False
+fp16_model: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8

code/configs/gptrm1b_from_sft_pythia410m.yml

@@ -0,0 +1,18 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 4
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: adamw_torch
+num_train_epochs: 1
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+padding: do_not_pad
+eval_steps: 0.25

code/configs/gptrm1b_pythia160m_r8_alpha32.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia1b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia160m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 16
+per_device_eval_batch_size: 16
+gradient_accumulation_steps: 2
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: adamw_torch
+num_train_epochs: 3
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32

code/configs/gptrm1b_pythia410m_fp16_r8_alpha32_4V100.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia1b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp16_trainall_3epochs
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 4
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: adamw_torch
+num_train_epochs: 1
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32

code/configs/gptrm1b_pythia410m_fp16_r8_alpha32_4V100_nopad.yml

@@ -0,0 +1,18 @@
+model_name: mnoukhov/pythia410m-tldr-sft
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt_relabel1b
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 4
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: adamw_torch
+num_train_epochs: 1
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+padding: do_not_pad
+eval_steps: 0.25

code/configs/gptrm1b_pythia410m_r8_alpha32_4V100.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia1b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32_trainall_3epochs
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 4
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: adamw_torch
+num_train_epochs: 3
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32

code/configs/gptrm_eval_adapter.yml

@@ -0,0 +1,11 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia7b_4V100_seq550
+pretrained_adapter: /home/toolkit/huggingface/tldr_gptrm_sft_pythia7b_4V100_seq560_fp16_3epochs_loralinear_adapter
+train_split: train[:1]
+eval_split: test[:5000]
+just_eval: True
+use_lora: False
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 4
+seq_length: 560

code/configs/gptrm_eval_pythia1b_fp16.yml

@@ -0,0 +1,10 @@
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+just_eval: True
+use_lora: False
+train_split: train[:1]
+eval_split: test[:5000]
+load_in_8bit: False
+fp16_model: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 8

code/configs/gptrm_eval_sft_pythia7b_seq560_fp16_3epochs_loralinear.yml

@@ -0,0 +1,10 @@
+model_name: /home/toolkit/huggingface/tldr_gptrm_sft_pythia7b_4V100_seq560_fp16_3epochs_loralinear
+train_split: train[:1]
+eval_split: test[:5000]
+just_eval: True
+use_lora: False
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_eval_batch_size: 4
+seq_length: 560

code/configs/gptrm_eval_sft_pythia7b_seq560_fp16_3epochs_loralinear_int8.yml

@@ -0,0 +1,10 @@
+model_name: /home/toolkit/huggingface/tldr_gptrm_sft_pythia7b_4V100_seq560_fp16_3epochs_loralinear
+just_eval: True
+use_lora: False
+train_split: train[:1]
+eval_split: test[:5000]
+load_in_8bit: True
+bf16: False
+fp16: False
+per_device_eval_batch_size: 4
+seq_length: 560

code/configs/gptrm_label_generated_pythia1b.yml

@@ -0,0 +1,12 @@
+mode: relabel
+output_dir: /home/toolkit/huggingface/openai_summarize_generated_20k_relabel_1b_margin
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+dataset_name: mnoukhov/openai_summarize_generated_20k
+train_split: train
+eval_split: null
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_eval_batch_size: 8
+padding: do_not_pad

code/configs/gptrm_label_pythia1b.yml

@@ -0,0 +1,12 @@
+mode: relabel
+output_dir: /home/toolkit/huggingface/openai_summarize_comparisons_tldrprompt_relabel1b_margin
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+dataset_name: mnoukhov/openai_summarize_comparisons_tldrprompt
+train_split: train
+eval_split: test[:5000]
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_eval_batch_size: 8
+padding: do_not_pad

code/configs/gptrm_predict_generated_pythia1b.yml

@@ -0,0 +1,12 @@
+mode: predict
+output_dir: /home/toolkit/huggingface/openai_summarize_generated_20k_relabel_1b_margin
+model_name: mnoukhov/pythia1b-sft-rm-tldrprompt
+dataset_name: mnoukhov/openai_summarize_generated_20k
+train_split: train
+eval_split: null
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_eval_batch_size: 8
+padding: do_not_pad

code/configs/gptrm_pythia1b_trainall_fp16_4V100.yml

@@ -0,0 +1,15 @@
+model_name: EleutherAI/pythia-1b-deduped
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 8
+gradient_checkpointing: False
+learning_rate: 1e-6
+optimizer_type: adamw_torch
+lr_scheduler_type: linear
+num_train_epochs: 3
+seq_length: 560
+eval_steps: 250

code/configs/gptrm_sft_pythia14m_fp32.yml

@@ -0,0 +1,15 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia410m
+model_name: /home/toolkit/huggingface/tldr_sft_pythia14m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 128
+per_device_eval_batch_size: 32
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 1
+seq_length: 560
+eval_steps: 0.25

code/configs/gptrm_sft_pythia14m_fp32_2epochs.yml

@@ -0,0 +1,15 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia410m
+model_name: /home/toolkit/huggingface/tldr_sft_pythia14m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 128
+per_device_eval_batch_size: 32
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 2
+seq_length: 560
+eval_steps: 0.2

code/configs/gptrm_sft_pythia14m_fp32_3epochs.yml

@@ -0,0 +1,15 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia410m
+model_name: /home/toolkit/huggingface/tldr_sft_pythia14m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 128
+per_device_eval_batch_size: 32
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+eval_steps: 0.1

code/configs/gptrm_sft_pythia14m_fp32_3epochs_lr1e-6.yml

@@ -0,0 +1,15 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia410m
+model_name: /home/toolkit/huggingface/tldr_sft_pythia14m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 128
+per_device_eval_batch_size: 32
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-6
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+eval_steps: 0.1

code/configs/gptrm_sft_pythia1b_fp16_lora_4V100.yml

@@ -0,0 +1,16 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia1b_fp16_trainall_3epochs
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 4
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: adamw_torch
+num_train_epochs: 3
+seq_length: 560
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32

code/configs/gptrm_sft_pythia1b_trainall_fp16_4V100.yml

@@ -0,0 +1,16 @@
+model_name: mnoukhov/pythia1b-tldr-sft
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 4
+gradient_accumulation_steps: 8
+gradient_checkpointing: False
+learning_rate: 1e-6
+optimizer_type: adamw_torch
+lr_scheduler_type: linear
+num_train_epochs: 2
+seq_length: 560
+eval_steps: 250
+padding: do_not_pad

code/configs/gptrm_sft_pythia1b_traincomp_fp16_4V100.yml

@@ -0,0 +1,15 @@
+model_name: /home/toolkit/trl/results/d31ad1f5a2ea94087cf49d8046228e62/code/results/checkpoint-1000
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 4
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 8
+gradient_checkpointing: False
+learning_rate: 1e-6
+optimizer_type: adamw_torch
+lr_scheduler_type: linear
+num_train_epochs: 3
+seq_length: 560
+eval_steps: 250

code/configs/gptrm_sft_pythia410m3_fp32_4V100.yml

@@ -0,0 +1,13 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32_trainall_3epochs
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 2
+gradient_checkpointing: False
+learning_rate: 5e-6
+optimizer_type: adamw_torch
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_amp16.yml

@@ -0,0 +1,13 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_amp16_batch128.yml

@@ -0,0 +1,14 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_amp16_batch8.yml

@@ -0,0 +1,14 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_fp16_loralinear_r32.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+use_lora: True
+lora_all_linear: True
+lora_r: 32

code/configs/gptrm_sft_pythia410m_fp16_loralinear_r8.yml

@@ -0,0 +1,17 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+use_lora: True
+lora_dropout: 0.01
+lora_all_linear: True
+lora_r: 8

code/configs/gptrm_sft_pythia410m_fp16_loralinear_r8_alpha32_batch128.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32

code/configs/gptrm_sft_pythia410m_fp32.yml

@@ -0,0 +1,14 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_fp32_adamtorch.yml

@@ -0,0 +1,14 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: adamw_torch
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_fp32_batch8.yml

@@ -0,0 +1,14 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+use_lora: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560

code/configs/gptrm_sft_pythia410m_fp32_loralinear_r32_a16_batch8.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 1
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+lora_all_linear: True
+lora_r: 32
+lora_alpha: 16

code/configs/gptrm_sft_pythia410m_fp32_loralinear_r8_alpha32_batch128.yml

@@ -0,0 +1,16 @@
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia7b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp32
+load_in_8bit: False
+bf16: False
+fp16: False
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32

code/configs/gptrm_sft_pythia7b_4V100_seq550_fp16.yml

@@ -0,0 +1,11 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia7b_4V100_seq550
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 2
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 2

code/configs/gptrm_sft_pythia7b_4V100_seq560_fp16_3epochs.yml

@@ -0,0 +1,12 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia7b_4V100_seq550
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 2
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 16
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 5
+seq_length: 560

code/configs/gptrm_sft_pythia7b_4V100_seq560_fp16_3epochs_loralinear.yml

@@ -0,0 +1,13 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia7b_4V100_seq550
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 1
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 32
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+lora_all_linear: True

code/configs/gptrm_sft_pythia7b_4V100_seq560_fp16_3epochs_loralinear_lr2e5_pagedadamw.yml

@@ -0,0 +1,13 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia7b_4V100_seq550
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 1
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 32
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 560
+lora_all_linear: True

code/configs/gptrm_sft_pythia7b_4V100_seq640_fp16_3epochs.yml

@@ -0,0 +1,12 @@
+model_name: /home/toolkit/huggingface/tldr_sft_pythia7b_4V100_seq550
+load_in_8bit: False
+bf16: False
+fp16: True
+per_device_train_batch_size: 1
+per_device_eval_batch_size: 2
+gradient_accumulation_steps: 32
+gradient_checkpointing: False
+learning_rate: 1e-5
+optimizer_type: paged_adamw_32bit
+num_train_epochs: 3
+seq_length: 640

code/configs/gptrm_test.yml

@@ -0,0 +1,20 @@
+train_split: train[:1000]
+eval_split: test[:1000]
+# model_name: EleutherAI/pythia-14m
+dataset_name: mnoukhov/openai_summarize_comparisons_relabel_pythia1b
+model_name: /home/toolkit/huggingface/tldr_sft_pythia410m_fp16_trainall_3epochs
+load_in_8bit: True
+bf16: False
+fp16: True
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+gradient_accumulation_steps: 4
+gradient_checkpointing: False
+learning_rate: 2e-5
+optimizer_type: adamw_torch
+num_train_epochs: 1
+use_lora: True
+lora_all_linear: True
+lora_r: 8
+lora_alpha: 32
+padding: do_not_pad

code/configs/newdpo2_pythia2.8b_tldr.yml

@@ -0,0 +1,29 @@
+## model and dataset
+model_name: mnoukhov/EleutherAI_pythia-2.8b-deduped__sft__tldr_55513 
+# hub_model_id: "mnoukhov/pythia-2.8b-dpo_hh_rlhf"
+dataset_name: mnoukhov/summarize_from_feedback_tldr3_unlabelled_vllm_dpo_costa_2.8b_bf16.yml_6e799_new
+eval_dataset_name: vwxyzjn/summarize_from_feedback_oai_preprocessing_1706381144
+dataset_eval_split: validation
+report_to: "wandb"
+## dpo
+learning_rate: 1e-5
+lr_scheduler_type: cosine
+fp16: False
+bf16: True
+gradient_accumulation_steps: 8
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 4
+num_train_epochs: 1
+max_length: 640
+max_prompt_length: 512 
+max_target_length: 128
+beta: 0.05
+## peft
+use_peft: True
+lora_r: 16
+lora_alpha: 32
+gradient_checkpointing: False
+evaluation_strategy: "steps"
+eval_steps: 0.2
+logging_steps: 100
+ddp_find_unused_parameters: False