model: model_args: device_map: cuda pretrained_model_name_or_path: saves/unlearn/tofu_Llama-3.2-3B-Instruct_forget10_GradDiff attn_implementation: flash_attention_2 torch_dtype: bfloat16 tokenizer_args: pretrained_model_name_or_path: meta-llama/Llama-3.2-3B-Instruct template_args: apply_chat_template: true system_prompt: You are a helpful assistant. system_prompt_with_special_tokens: '<|begin_of_text|><|start_header_id|>system<|end_header_id|> You are a helpful assistant.<|eot_id|>' user_start_tag: '<|start_header_id|>user<|end_header_id|> ' user_end_tag: <|eot_id|> asst_start_tag: '<|start_header_id|>assistant<|end_header_id|> ' asst_end_tag: <|eot_id|> date_string: 10 Apr 2025 mode: eval task_name: tofu_Llama-3.2-3B-Instruct_forget10_GradDiff seed: 0 eval: tofu: metrics: forget_quality: pre_compute: forget_truth_ratio: pre_compute: forget_Q_A_PARA_Prob: datasets: TOFU_QA_forget_para: handler: QADataset args: hf_args: name: ${eval.tofu.forget_split}_perturbed split: train path: locuslab/TOFU question_key: question answer_key: paraphrased_answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 access_key: correct forget_Q_A_PERT_Prob: datasets: TOFU_QA_forget_pert: handler: QADataset args: hf_args: name: ${eval.tofu.forget_split}_perturbed split: train path: locuslab/TOFU question_key: question answer_key: perturbed_answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 access_key: wrong handler: truth_ratio aggregator: closer_to_1_better access_key: forget reference_logs: retain_model_logs: path: ${eval.tofu.retain_logs_path} include: forget_truth_ratio: access_key: retain handler: ks_test forget_Q_A_Prob: datasets: TOFU_QA_forget: handler: QADataset args: hf_args: name: ${eval.tofu.forget_split} split: train path: locuslab/TOFU question_key: question answer_key: answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 forget_Q_A_ROUGE: datasets: TOFU_QA_forget: handler: QADataset args: hf_args: name: ${eval.tofu.forget_split} split: train path: locuslab/TOFU question_key: question answer_key: answer max_length: 512 predict_with_generate: true collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: left index: index generation_args: do_sample: false top_p: null temperature: null max_new_tokens: 200 use_cache: true handler: rouge rouge_type: rougeL_recall batch_size: 32 model_utility: pre_compute: retain_Q_A_Prob: datasets: TOFU_QA_retain_eval: handler: QADataset args: hf_args: name: retain_perturbed split: train path: locuslab/TOFU question_key: question answer_key: answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 retain_Q_A_ROUGE: datasets: TOFU_QA_retain_eval: handler: QADataset args: hf_args: name: retain_perturbed split: train path: locuslab/TOFU question_key: question answer_key: answer max_length: 512 predict_with_generate: true collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: left index: index generation_args: do_sample: false top_p: null temperature: null max_new_tokens: 200 use_cache: true handler: rouge rouge_type: rougeL_recall batch_size: 32 retain_Truth_Ratio: pre_compute: retain_Q_A_PARA_Prob: datasets: TOFU_QA_retain_para: handler: QADataset args: hf_args: name: retain_perturbed split: train path: locuslab/TOFU question_key: question answer_key: paraphrased_answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 access_key: correct retain_Q_A_PERT_Prob: datasets: TOFU_QA_retain_pert: handler: QADataset args: hf_args: name: retain_perturbed split: train path: locuslab/TOFU question_key: question answer_key: perturbed_answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 access_key: wrong handler: truth_ratio aggregator: true_better ra_Q_A_Prob_normalised: pre_compute: ra_Q_A_Prob: datasets: TOFU_QA_ra: handler: QADataset args: hf_args: name: real_authors_perturbed split: train path: locuslab/TOFU question_key: question answer_key: answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 access_key: correct ra_Q_A_PERT_Prob: datasets: TOFU_QA_ra_pert: handler: QADataset args: hf_args: name: real_authors_perturbed split: train path: locuslab/TOFU question_key: question answer_key: perturbed_answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 access_key: wrong handler: probability_w_options ra_Q_A_ROUGE: datasets: TOFU_QA_ra: handler: QADataset args: hf_args: name: real_authors_perturbed split: train path: locuslab/TOFU question_key: question answer_key: answer max_length: 512 predict_with_generate: true collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: left index: index generation_args: do_sample: false top_p: null temperature: null max_new_tokens: 200 use_cache: true handler: rouge rouge_type: rougeL_recall batch_size: 32 ra_Truth_Ratio: pre_compute: ra_Q_A_Prob: datasets: TOFU_QA_ra: handler: QADataset args: hf_args: name: real_authors_perturbed split: train path: locuslab/TOFU question_key: question answer_key: answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 access_key: correct ra_Q_A_PERT_Prob: datasets: TOFU_QA_ra_pert: handler: QADataset args: hf_args: name: real_authors_perturbed split: train path: locuslab/TOFU question_key: question answer_key: perturbed_answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 access_key: wrong handler: truth_ratio aggregator: true_better wf_Q_A_Prob_normalised: pre_compute: wf_Q_A_Prob: datasets: TOFU_QA_wf: handler: QADataset args: hf_args: name: world_facts_perturbed split: train path: locuslab/TOFU question_key: question answer_key: answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 access_key: correct wf_Q_A_PERT_Prob: datasets: TOFU_QA_wf_pert: handler: QADataset args: hf_args: name: world_facts_perturbed split: train path: locuslab/TOFU question_key: question answer_key: perturbed_answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 access_key: wrong handler: probability_w_options wf_Q_A_ROUGE: datasets: TOFU_QA_wf: handler: QADataset args: hf_args: name: world_facts_perturbed split: train path: locuslab/TOFU question_key: question answer_key: answer max_length: 512 predict_with_generate: true collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: left index: index generation_args: do_sample: false top_p: null temperature: null max_new_tokens: 200 use_cache: true handler: rouge rouge_type: rougeL_recall batch_size: 32 wf_Truth_Ratio: pre_compute: wf_Q_A_Prob: datasets: TOFU_QA_wf: handler: QADataset args: hf_args: name: world_facts_perturbed split: train path: locuslab/TOFU question_key: question answer_key: answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 access_key: correct wf_Q_A_PERT_Prob: datasets: TOFU_QA_wf_pert: handler: QADataset args: hf_args: name: world_facts_perturbed split: train path: locuslab/TOFU question_key: question answer_key: perturbed_answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: probability batch_size: 32 access_key: wrong handler: truth_ratio aggregator: true_better handler: hm_aggregate privleak: pre_compute: mia_min_k: datasets: TOFU_QA_forget: access_key: forget handler: QADataset args: hf_args: name: ${eval.tofu.forget_split} split: train path: locuslab/TOFU question_key: question answer_key: answer max_length: 512 TOFU_QA_holdout: access_key: holdout handler: QADataset args: hf_args: name: ${eval.tofu.holdout_split} path: locuslab/TOFU split: train question_key: question answer_key: answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index batch_size: 32 handler: mia_min_k k: 0.4 access_key: forget reference_logs: retain_model_logs: path: ${eval.tofu.retain_logs_path} include: mia_min_k: access_key: retain handler: privleak ref_value: 0.5 extraction_strength: datasets: TOFU_QA_forget: handler: QADataset args: hf_args: name: ${eval.tofu.forget_split} split: train path: locuslab/TOFU question_key: question answer_key: answer max_length: 512 collators: DataCollatorForSupervisedDataset: handler: DataCollatorForSupervisedDataset args: padding_side: right index: index handler: extraction_strength batch_size: 32 handler: TOFUEvaluator output_dir: ${paths.output_dir} overwrite: false forget_split: ${forget_split} holdout_split: ${holdout_split} retain_logs_path: ${retain_logs_path} paths: root_dir: . data_dir: ${paths.root_dir}/data/ datasets: ${paths.root_dir}/configs/data/datasets output_dir: saves/unlearn/tofu_Llama-3.2-3B-Instruct_forget10_GradDiff/evals work_dir: ${hydra:runtime.cwd} forget_split: forget10 holdout_split: holdout10 retain_logs_path: saves/eval/tofu_Llama-3.2-3B-Instruct_retain90/TOFU_EVAL.json