初始化项目，由ModelHub XC社区提供模型
Model: JoaoBoer/tofu_Llama-3.1-8B-Instruct_forget05_NPO Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
824
.hydra/config.yaml
Normal file
824
.hydra/config.yaml
Normal file
@@ -0,0 +1,824 @@
|
||||
model:
|
||||
model_args:
|
||||
pretrained_model_name_or_path: open-unlearning/tofu_Llama-3.1-8B-Instruct_full
|
||||
attn_implementation: flash_attention_2
|
||||
torch_dtype: bfloat16
|
||||
tokenizer_args:
|
||||
pretrained_model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
|
||||
template_args:
|
||||
apply_chat_template: true
|
||||
system_prompt: You are a helpful assistant.
|
||||
system_prompt_with_special_tokens: '<|begin_of_text|><|start_header_id|>system<|end_header_id|>
|
||||
|
||||
|
||||
You are a helpful assistant.<|eot_id|>'
|
||||
user_start_tag: '<|start_header_id|>user<|end_header_id|>
|
||||
|
||||
|
||||
'
|
||||
user_end_tag: <|eot_id|>
|
||||
asst_start_tag: '<|start_header_id|>assistant<|end_header_id|>
|
||||
|
||||
|
||||
'
|
||||
asst_end_tag: <|eot_id|>
|
||||
date_string: 10 Apr 2025
|
||||
trainer:
|
||||
handler: NPO
|
||||
args:
|
||||
per_device_train_batch_size: 4
|
||||
per_device_eval_batch_size: 16
|
||||
gradient_accumulation_steps: 4
|
||||
learning_rate: 2.0e-05
|
||||
bf16: true
|
||||
bf16_full_eval: true
|
||||
logging_steps: 5
|
||||
output_dir: ${paths.output_dir}
|
||||
logging_dir: ${trainer.args.output_dir}/logs
|
||||
log_on_each_node: false
|
||||
report_to: tensorboard
|
||||
ddp_find_unused_parameters: true
|
||||
gradient_checkpointing: true
|
||||
optim: paged_adamw_32bit
|
||||
save_strategy: 'no'
|
||||
save_only_model: true
|
||||
weight_decay: 0.01
|
||||
do_train: true
|
||||
do_eval: false
|
||||
eval_on_start: false
|
||||
eval_strategy: 'no'
|
||||
num_train_epochs: 10
|
||||
seed: 0
|
||||
deepspeed: configs/accelerate/zero_stage3_offload_config.json
|
||||
warmup_epochs: 1.0
|
||||
remove_unused_columns: false
|
||||
method_args:
|
||||
gamma: 1.0
|
||||
alpha: 2
|
||||
retain_loss_type: NLL
|
||||
beta: 0.1
|
||||
data:
|
||||
forget:
|
||||
TOFU_QA_forget:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${forget_split}
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
retain:
|
||||
TOFU_QA_retain:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${retain_split}
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
anchor: forget
|
||||
collator:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
eval:
|
||||
tofu:
|
||||
metrics:
|
||||
forget_quality:
|
||||
pre_compute:
|
||||
forget_truth_ratio:
|
||||
pre_compute:
|
||||
forget_Q_A_PARA_Prob:
|
||||
datasets:
|
||||
TOFU_QA_forget_para:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: paraphrased_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
forget_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_forget_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: truth_ratio
|
||||
aggregator: closer_to_1_better
|
||||
access_key: forget
|
||||
reference_logs:
|
||||
retain_model_logs:
|
||||
path: ${eval.tofu.retain_logs_path}
|
||||
include:
|
||||
forget_truth_ratio:
|
||||
access_key: retain
|
||||
handler: ks_test
|
||||
model_utility:
|
||||
pre_compute:
|
||||
retain_Q_A_Prob:
|
||||
datasets:
|
||||
TOFU_QA_retain_eval:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: retain_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
retain_Q_A_ROUGE:
|
||||
datasets:
|
||||
TOFU_QA_retain_eval:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: retain_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
predict_with_generate: true
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: left
|
||||
index: index
|
||||
generation_args:
|
||||
do_sample: false
|
||||
top_p: null
|
||||
temperature: null
|
||||
max_new_tokens: 200
|
||||
use_cache: true
|
||||
handler: rouge
|
||||
rouge_type: rougeL_recall
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
retain_Truth_Ratio:
|
||||
pre_compute:
|
||||
retain_Q_A_PARA_Prob:
|
||||
datasets:
|
||||
TOFU_QA_retain_para:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: retain_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: paraphrased_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
retain_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_retain_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: retain_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: truth_ratio
|
||||
aggregator: true_better
|
||||
ra_Q_A_Prob_normalised:
|
||||
pre_compute:
|
||||
ra_Q_A_Prob:
|
||||
datasets:
|
||||
TOFU_QA_ra:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: real_authors_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
ra_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_ra_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: real_authors_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: probability_w_options
|
||||
ra_Q_A_ROUGE:
|
||||
datasets:
|
||||
TOFU_QA_ra:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: real_authors_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
predict_with_generate: true
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: left
|
||||
index: index
|
||||
generation_args:
|
||||
do_sample: false
|
||||
top_p: null
|
||||
temperature: null
|
||||
max_new_tokens: 200
|
||||
use_cache: true
|
||||
handler: rouge
|
||||
rouge_type: rougeL_recall
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
ra_Truth_Ratio:
|
||||
pre_compute:
|
||||
ra_Q_A_Prob:
|
||||
datasets:
|
||||
TOFU_QA_ra:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: real_authors_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
ra_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_ra_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: real_authors_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: truth_ratio
|
||||
aggregator: true_better
|
||||
wf_Q_A_Prob_normalised:
|
||||
pre_compute:
|
||||
wf_Q_A_Prob:
|
||||
datasets:
|
||||
TOFU_QA_wf:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: world_facts_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
wf_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_wf_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: world_facts_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: probability_w_options
|
||||
wf_Q_A_ROUGE:
|
||||
datasets:
|
||||
TOFU_QA_wf:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: world_facts_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
predict_with_generate: true
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: left
|
||||
index: index
|
||||
generation_args:
|
||||
do_sample: false
|
||||
top_p: null
|
||||
temperature: null
|
||||
max_new_tokens: 200
|
||||
use_cache: true
|
||||
handler: rouge
|
||||
rouge_type: rougeL_recall
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
wf_Truth_Ratio:
|
||||
pre_compute:
|
||||
wf_Q_A_Prob:
|
||||
datasets:
|
||||
TOFU_QA_wf:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: world_facts_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
wf_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_wf_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: world_facts_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: truth_ratio
|
||||
aggregator: true_better
|
||||
handler: hm_aggregate
|
||||
exact_memorization:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: exact_memorization
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
extraction_strength:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: extraction_strength
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
forget_Q_A_PARA_Prob:
|
||||
datasets:
|
||||
TOFU_QA_forget_para:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: paraphrased_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
forget_truth_ratio:
|
||||
pre_compute:
|
||||
forget_Q_A_PARA_Prob:
|
||||
datasets:
|
||||
TOFU_QA_forget_para:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: paraphrased_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
forget_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_forget_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: truth_ratio
|
||||
aggregator: closer_to_1_better
|
||||
forget_Q_A_gibberish:
|
||||
pre_compute:
|
||||
forget_Q_A_ROUGE:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
predict_with_generate: true
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: left
|
||||
index: index
|
||||
generation_args:
|
||||
do_sample: false
|
||||
top_p: null
|
||||
temperature: null
|
||||
max_new_tokens: 200
|
||||
use_cache: true
|
||||
handler: rouge
|
||||
rouge_type: rougeL_recall
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: text
|
||||
handler: classifier_prob
|
||||
batch_size: 32
|
||||
max_length: 32
|
||||
class_id: 0
|
||||
text_key: generation
|
||||
device: cuda
|
||||
classifier_model_args:
|
||||
pretrained_model_name_or_path: madhurjindal/autonlp-Gibberish-Detector-492513457
|
||||
classifier_tokenization_args:
|
||||
pretrained_model_name_or_path: madhurjindal/autonlp-Gibberish-Detector-492513457
|
||||
privleak:
|
||||
pre_compute:
|
||||
mia_min_k:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
access_key: forget
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
TOFU_QA_holdout:
|
||||
access_key: holdout
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.holdout_split}
|
||||
path: locuslab/TOFU
|
||||
split: train
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
handler: mia_min_k
|
||||
k: 0.4
|
||||
access_key: forget
|
||||
reference_logs:
|
||||
retain_model_logs:
|
||||
path: ${eval.tofu.retain_logs_path}
|
||||
include:
|
||||
mia_min_k:
|
||||
access_key: retain
|
||||
handler: privleak
|
||||
ref_value: 0.5
|
||||
mia_min_k_plus_plus:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
access_key: forget
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
TOFU_QA_holdout:
|
||||
access_key: holdout
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.holdout_split}
|
||||
path: locuslab/TOFU
|
||||
split: train
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
k: 0.4
|
||||
handler: mia_min_k_plus_plus
|
||||
mia_min_k:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
access_key: forget
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
TOFU_QA_holdout:
|
||||
access_key: holdout
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.holdout_split}
|
||||
path: locuslab/TOFU
|
||||
split: train
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
handler: mia_min_k
|
||||
k: 0.4
|
||||
mia_loss:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
access_key: forget
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
TOFU_QA_holdout:
|
||||
access_key: holdout
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.holdout_split}
|
||||
path: locuslab/TOFU
|
||||
split: train
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
handler: mia_loss
|
||||
mia_zlib:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
access_key: forget
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
TOFU_QA_holdout:
|
||||
access_key: holdout
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.holdout_split}
|
||||
path: locuslab/TOFU
|
||||
split: train
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
handler: mia_zlib
|
||||
handler: TOFUEvaluator
|
||||
output_dir: ${paths.output_dir}
|
||||
overwrite: true
|
||||
forget_split: ${forget_split}
|
||||
holdout_split: ${holdout_split}
|
||||
retain_logs_path: ${retain_logs_path}
|
||||
question_key: ${question_key}
|
||||
batch_size: 32
|
||||
paths:
|
||||
root_dir: .
|
||||
data_dir: ${paths.root_dir}/data/
|
||||
datasets: ${paths.root_dir}/configs/data/datasets
|
||||
output_dir: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
|
||||
work_dir: ${hydra:runtime.cwd}
|
||||
forget_split: forget05
|
||||
retain_split: retain95
|
||||
holdout_split: holdout05
|
||||
retain_logs_path: saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
|
||||
question_key: question
|
||||
task_name: tofu_Llama-3.1-8B-Instruct_forget05_NPO_lr2e-05_beta0.1_alpha2_epoch10
|
||||
mode: unlearn
|
||||
309
.hydra/hydra.yaml
Normal file
309
.hydra/hydra.yaml
Normal file
@@ -0,0 +1,309 @@
|
||||
hydra:
|
||||
run:
|
||||
dir: ${paths.output_dir}
|
||||
sweep:
|
||||
dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
|
||||
subdir: ${hydra.job.num}
|
||||
launcher:
|
||||
_target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
|
||||
sweeper:
|
||||
_target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
|
||||
max_batch_size: null
|
||||
params: null
|
||||
help:
|
||||
app_name: ${hydra.job.name}
|
||||
header: '${hydra.help.app_name} is powered by Hydra.
|
||||
|
||||
'
|
||||
footer: 'Powered by Hydra (https://hydra.cc)
|
||||
|
||||
Use --hydra-help to view Hydra specific help
|
||||
|
||||
'
|
||||
template: '${hydra.help.header}
|
||||
|
||||
== Configuration groups ==
|
||||
|
||||
Compose your configuration from those groups (group=option)
|
||||
|
||||
|
||||
$APP_CONFIG_GROUPS
|
||||
|
||||
|
||||
== Config ==
|
||||
|
||||
Override anything in the config (foo.bar=value)
|
||||
|
||||
|
||||
$CONFIG
|
||||
|
||||
|
||||
${hydra.help.footer}
|
||||
|
||||
'
|
||||
hydra_help:
|
||||
template: 'Hydra (${hydra.runtime.version})
|
||||
|
||||
See https://hydra.cc for more info.
|
||||
|
||||
|
||||
== Flags ==
|
||||
|
||||
$FLAGS_HELP
|
||||
|
||||
|
||||
== Configuration groups ==
|
||||
|
||||
Compose your configuration from those groups (For example, append hydra/job_logging=disabled
|
||||
to command line)
|
||||
|
||||
|
||||
$HYDRA_CONFIG_GROUPS
|
||||
|
||||
|
||||
Use ''--cfg hydra'' to Show the Hydra config.
|
||||
|
||||
'
|
||||
hydra_help: ???
|
||||
hydra_logging:
|
||||
version: 1
|
||||
formatters:
|
||||
colorlog:
|
||||
(): colorlog.ColoredFormatter
|
||||
format: '[%(cyan)s%(asctime)s%(reset)s][%(purple)sHYDRA%(reset)s] %(message)s'
|
||||
handlers:
|
||||
console:
|
||||
class: logging.StreamHandler
|
||||
formatter: colorlog
|
||||
stream: ext://sys.stdout
|
||||
root:
|
||||
level: INFO
|
||||
handlers:
|
||||
- console
|
||||
disable_existing_loggers: false
|
||||
job_logging:
|
||||
version: 1
|
||||
formatters:
|
||||
simple:
|
||||
format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
|
||||
colorlog:
|
||||
(): colorlog.ColoredFormatter
|
||||
format: '[%(cyan)s%(asctime)s%(reset)s][%(blue)s%(name)s%(reset)s][%(log_color)s%(levelname)s%(reset)s]
|
||||
- %(message)s'
|
||||
log_colors:
|
||||
DEBUG: purple
|
||||
INFO: green
|
||||
WARNING: yellow
|
||||
ERROR: red
|
||||
CRITICAL: red
|
||||
handlers:
|
||||
console:
|
||||
class: logging.StreamHandler
|
||||
formatter: colorlog
|
||||
stream: ext://sys.stdout
|
||||
file:
|
||||
class: logging.FileHandler
|
||||
formatter: simple
|
||||
filename: ${hydra.runtime.output_dir}/${trainer.handler}.log
|
||||
root:
|
||||
level: INFO
|
||||
handlers:
|
||||
- console
|
||||
- file
|
||||
disable_existing_loggers: false
|
||||
env: {}
|
||||
mode: RUN
|
||||
searchpath: []
|
||||
callbacks: {}
|
||||
output_subdir: .hydra
|
||||
overrides:
|
||||
hydra:
|
||||
- hydra.mode=RUN
|
||||
task:
|
||||
- experiment=unlearn/tofu/default.yaml
|
||||
- trainer=NPO
|
||||
- task_name=tofu_Llama-3.1-8B-Instruct_forget05_NPO_lr2e-05_beta0.1_alpha2_epoch10
|
||||
- model=Llama-3.1-8B-Instruct
|
||||
- forget_split=forget05
|
||||
- holdout_split=holdout05
|
||||
- retain_split=retain95
|
||||
- model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.1-8B-Instruct_full
|
||||
- retain_logs_path=saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
|
||||
- paths.output_dir=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
|
||||
- trainer.args.learning_rate=2e-05
|
||||
- trainer.args.num_train_epochs=10
|
||||
- trainer.args.per_device_train_batch_size=4
|
||||
- trainer.args.gradient_accumulation_steps=4
|
||||
- trainer.args.ddp_find_unused_parameters=true
|
||||
- trainer.args.gradient_checkpointing=true
|
||||
- trainer.args.eval_strategy=no
|
||||
- trainer.args.eval_on_start=False
|
||||
- trainer.args.do_eval=False
|
||||
- trainer.method_args.beta=0.1
|
||||
- trainer.method_args.alpha=2
|
||||
- trainer.method_args.gamma=1.0
|
||||
- trainer.method_args.retain_loss_type=NLL
|
||||
job:
|
||||
name: train
|
||||
chdir: null
|
||||
override_dirname: experiment=unlearn/tofu/default.yaml,forget_split=forget05,holdout_split=holdout05,model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.1-8B-Instruct_full,model=Llama-3.1-8B-Instruct,paths.output_dir=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05,retain_logs_path=saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json,retain_split=retain95,task_name=tofu_Llama-3.1-8B-Instruct_forget05_NPO_lr2e-05_beta0.1_alpha2_epoch10,trainer.args.ddp_find_unused_parameters=true,trainer.args.do_eval=False,trainer.args.eval_on_start=False,trainer.args.eval_strategy=no,trainer.args.gradient_accumulation_steps=4,trainer.args.gradient_checkpointing=true,trainer.args.learning_rate=2e-05,trainer.args.num_train_epochs=10,trainer.args.per_device_train_batch_size=4,trainer.method_args.alpha=2,trainer.method_args.beta=0.1,trainer.method_args.gamma=1.0,trainer.method_args.retain_loss_type=NLL,trainer=NPO
|
||||
id: ???
|
||||
num: ???
|
||||
config_name: unlearn.yaml
|
||||
env_set: {}
|
||||
env_copy: []
|
||||
config:
|
||||
override_dirname:
|
||||
kv_sep: '='
|
||||
item_sep: ','
|
||||
exclude_keys: []
|
||||
runtime:
|
||||
version: 1.3.0
|
||||
version_base: '1.3'
|
||||
cwd: /home/joaoabitante/speculative-decoding-unlearning
|
||||
config_sources:
|
||||
- path: hydra.conf
|
||||
schema: pkg
|
||||
provider: hydra
|
||||
- path: /home/joaoabitante/speculative-decoding-unlearning/configs
|
||||
schema: file
|
||||
provider: main
|
||||
- path: hydra_plugins.hydra_colorlog.conf
|
||||
schema: pkg
|
||||
provider: hydra-colorlog
|
||||
- path: ''
|
||||
schema: structured
|
||||
provider: schema
|
||||
output_dir: /home/joaoabitante/speculative-decoding-unlearning/saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
|
||||
choices:
|
||||
adapter: null
|
||||
quantization: null
|
||||
experiment: unlearn/tofu/default.yaml
|
||||
paths: default
|
||||
hydra: default
|
||||
eval: tofu
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.mia_zlib.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.mia_zlib.datasets: TOFU_MIA
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.mia_loss.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.mia_loss.datasets: TOFU_MIA
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.mia_min_k.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.mia_min_k.datasets: TOFU_MIA
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.mia_min_k_plus_plus.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.mia_min_k_plus_plus.datasets: TOFU_MIA
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.privleak.pre_compute.mia_min_k: mia_min_k
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.privleak.pre_compute.mia_min_k.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.privleak.pre_compute.mia_min_k.datasets: TOFU_MIA
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.forget_Q_A_gibberish.pre_compute.forget_Q_A_ROUGE: forget_Q_A_ROUGE
|
||||
eval/tofu_metrics/./../../generation@eval.tofu.metrics.forget_Q_A_gibberish.pre_compute.forget_Q_A_ROUGE.generation_args: default
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_Q_A_gibberish.pre_compute.forget_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_Q_A_gibberish.pre_compute.forget_Q_A_ROUGE.datasets: TOFU_QA_forget
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets: TOFU_QA_forget_pert
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets: TOFU_QA_forget_para
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_PARA_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_PARA_Prob.datasets: TOFU_QA_forget_para
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.extraction_strength.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.extraction_strength.datasets: TOFU_QA_forget
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.exact_memorization.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.exact_memorization.datasets: TOFU_QA_forget
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio: wf_Truth_Ratio
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.datasets
|
||||
: TOFU_QA_wf_pert
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.datasets
|
||||
: TOFU_QA_wf
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE: wf_Q_A_ROUGE
|
||||
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.generation_args: default
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.datasets: TOFU_QA_wf
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised: wf_Q_A_Prob_normalised
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.datasets
|
||||
: TOFU_QA_wf_pert
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.datasets
|
||||
: TOFU_QA_wf
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio: ra_Truth_Ratio
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.datasets
|
||||
: TOFU_QA_ra_pert
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.datasets
|
||||
: TOFU_QA_ra
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE: ra_Q_A_ROUGE
|
||||
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.generation_args: default
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.datasets: TOFU_QA_ra
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised: ra_Q_A_Prob_normalised
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.datasets
|
||||
: TOFU_QA_ra_pert
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.datasets
|
||||
: TOFU_QA_ra
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio: retain_Truth_Ratio
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob: retain_Q_A_PERT_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.datasets
|
||||
: TOFU_QA_retain_pert
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob: retain_Q_A_PARA_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.datasets
|
||||
: TOFU_QA_retain_para
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE: retain_Q_A_ROUGE
|
||||
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.generation_args: default
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.datasets: TOFU_QA_retain_eval
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob: retain_Q_A_Prob
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.datasets: TOFU_QA_retain_eval
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio: forget_Truth_Ratio
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets
|
||||
: TOFU_QA_forget_pert
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets
|
||||
: TOFU_QA_forget_para
|
||||
collator: DataCollatorForSupervisedDataset
|
||||
data: unlearn
|
||||
data/datasets@data.eval: null
|
||||
data/datasets@data.retain: TOFU_QA_retain
|
||||
data/datasets@data.forget: TOFU_QA_forget
|
||||
trainer: NPO
|
||||
model: Llama-3.1-8B-Instruct
|
||||
hydra/env: default
|
||||
hydra/callbacks: null
|
||||
hydra/job_logging: colorlog
|
||||
hydra/hydra_logging: colorlog
|
||||
hydra/hydra_help: default
|
||||
hydra/help: default
|
||||
hydra/sweeper: basic
|
||||
hydra/launcher: basic
|
||||
hydra/output: default
|
||||
verbose: false
|
||||
23
.hydra/overrides.yaml
Normal file
23
.hydra/overrides.yaml
Normal file
@@ -0,0 +1,23 @@
|
||||
- experiment=unlearn/tofu/default.yaml
|
||||
- trainer=NPO
|
||||
- task_name=tofu_Llama-3.1-8B-Instruct_forget05_NPO_lr2e-05_beta0.1_alpha2_epoch10
|
||||
- model=Llama-3.1-8B-Instruct
|
||||
- forget_split=forget05
|
||||
- holdout_split=holdout05
|
||||
- retain_split=retain95
|
||||
- model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.1-8B-Instruct_full
|
||||
- retain_logs_path=saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
|
||||
- paths.output_dir=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
|
||||
- trainer.args.learning_rate=2e-05
|
||||
- trainer.args.num_train_epochs=10
|
||||
- trainer.args.per_device_train_batch_size=4
|
||||
- trainer.args.gradient_accumulation_steps=4
|
||||
- trainer.args.ddp_find_unused_parameters=true
|
||||
- trainer.args.gradient_checkpointing=true
|
||||
- trainer.args.eval_strategy=no
|
||||
- trainer.args.eval_on_start=False
|
||||
- trainer.args.do_eval=False
|
||||
- trainer.method_args.beta=0.1
|
||||
- trainer.method_args.alpha=2
|
||||
- trainer.method_args.gamma=1.0
|
||||
- trainer.method_args.retain_loss_type=NLL
|
||||
18
NPO.log
Normal file
18
NPO.log
Normal file
@@ -0,0 +1,18 @@
|
||||
[2026-08-13 20:32:37,656][model][INFO] - Setting pad_token as eos token: <|eot_id|>
|
||||
[2026-08-13 20:32:37,656][__main__][INFO] - Enabling input_require_grads on LlamaForCausalLM for gradient checkpointing
|
||||
[2026-08-13 20:32:41,236][evaluator][INFO] - Evaluations stored in the experiment directory: saves/unlearn/baselines/tofu/NPO_lr2e-05_beta0.1_alpha2_epoch10/Llama-3.1-8B-Instruct/forget05
|
||||
[2026-08-13 20:34:24,404][trainer][INFO] - NPO Trainer loaded, output_dir: saves/unlearn/baselines/tofu/NPO_lr2e-05_beta0.1_alpha2_epoch10/Llama-3.1-8B-Instruct/forget05
|
||||
[2026-08-13 20:34:24,405][trainer][INFO] - [32mMethod args: {'gamma': 1.0, 'alpha': 2, 'retain_loss_type': 'NLL', 'beta': 0.1}[0m
|
||||
[2026-08-13 20:34:24,405][__main__][INFO] - [32mTraining setup: epochs=10, batch_size=4, learning_rate=2e-05, gradient_accumulation_steps=4, weight_decay=0.01[0m
|
||||
[2026-08-24 14:06:12,102][model][INFO] - Setting pad_token as eos token: <|eot_id|>
|
||||
[2026-08-24 14:06:12,103][__main__][INFO] - Enabling input_require_grads on LlamaForCausalLM for gradient checkpointing
|
||||
[2026-08-24 14:06:15,484][evaluator][INFO] - Evaluations stored in the experiment directory: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
|
||||
[2026-08-24 14:07:43,097][trainer][INFO] - NPO Trainer loaded, output_dir: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
|
||||
[2026-08-24 14:07:43,099][trainer][INFO] - [32mMethod args: {'gamma': 1.0, 'alpha': 2, 'retain_loss_type': 'NLL', 'beta': 0.1}[0m
|
||||
[2026-08-24 14:07:43,099][__main__][INFO] - [32mTraining setup: epochs=10, batch_size=4, learning_rate=2e-05, gradient_accumulation_steps=4, weight_decay=0.01[0m
|
||||
[2026-08-24 15:02:45,283][model][INFO] - Setting pad_token as eos token: <|eot_id|>
|
||||
[2026-08-24 15:02:45,284][__main__][INFO] - Enabling input_require_grads on LlamaForCausalLM for gradient checkpointing
|
||||
[2026-08-24 15:02:48,348][evaluator][INFO] - Evaluations stored in the experiment directory: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
|
||||
[2026-08-24 15:04:15,208][trainer][INFO] - NPO Trainer loaded, output_dir: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
|
||||
[2026-08-24 15:04:15,209][trainer][INFO] - [32mMethod args: {'gamma': 1.0, 'alpha': 2, 'retain_loss_type': 'NLL', 'beta': 0.1}[0m
|
||||
[2026-08-24 15:04:15,209][__main__][INFO] - [32mTraining setup: epochs=10, batch_size=4, learning_rate=2e-05, gradient_accumulation_steps=4, weight_decay=0.01[0m
|
||||
45
README.md
Normal file
45
README.md
Normal file
@@ -0,0 +1,45 @@
|
||||
---
|
||||
license: llama3.1
|
||||
base_model: open-unlearning/tofu_Llama-3.1-8B-Instruct_full
|
||||
library_name: transformers
|
||||
pipeline_tag: text-generation
|
||||
datasets:
|
||||
- locuslab/TOFU
|
||||
tags:
|
||||
- unlearning
|
||||
- tofu
|
||||
- NPO
|
||||
- forget05
|
||||
---
|
||||
|
||||
# tofu_Llama-3.1-8B-Instruct_forget05_NPO
|
||||
|
||||
`open-unlearning/tofu_Llama-3.1-8B-Instruct_full` unlearned on the TOFU `forget05` split with **NPO**, trained with the [open-unlearning](https://github.com/locuslab/open-unlearning) framework. Used as a weight-unlearning baseline / draft model in the [Speculative-Decoding-Unlearning](https://github.com/JoaoVitorBoer/Speculative-Decoding-Unlearning) project.
|
||||
|
||||
Full training config: `.hydra/config.yaml`. TOFU evaluation outputs: `evals/`.
|
||||
|
||||
## Method hyperparameters
|
||||
|
||||
```yaml
|
||||
gamma: 1.0
|
||||
alpha: 2
|
||||
retain_loss_type: NLL
|
||||
beta: 0.1
|
||||
```
|
||||
|
||||
## TOFU summary metrics
|
||||
|
||||
| metric | value |
|
||||
|---|---|
|
||||
| exact_memorization | 0.5073 |
|
||||
| extraction_strength | 0.0558 |
|
||||
| forget_Q_A_PARA_Prob | 0.0237 |
|
||||
| forget_Q_A_gibberish | 0.8973 |
|
||||
| forget_quality | 0.0118 |
|
||||
| forget_truth_ratio | 0.6632 |
|
||||
| mia_loss | 0.0650 |
|
||||
| mia_min_k | 0.0673 |
|
||||
| mia_min_k_plus_plus | 0.1206 |
|
||||
| mia_zlib | 0.1069 |
|
||||
| model_utility | 0.5679 |
|
||||
| privleak | 44.8684 |
|
||||
39
config.json
Normal file
39
config.json
Normal file
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"architectures": [
|
||||
"LlamaForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 128000,
|
||||
"eos_token_id": [
|
||||
128001,
|
||||
128008,
|
||||
128009
|
||||
],
|
||||
"head_dim": 128,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 4096,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 14336,
|
||||
"max_position_embeddings": 131072,
|
||||
"mlp_bias": false,
|
||||
"model_type": "llama",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 32,
|
||||
"num_key_value_heads": 8,
|
||||
"pretraining_tp": 1,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"rope_scaling": {
|
||||
"factor": 8.0,
|
||||
"high_freq_factor": 4.0,
|
||||
"low_freq_factor": 1.0,
|
||||
"original_max_position_embeddings": 8192,
|
||||
"rope_type": "llama3"
|
||||
},
|
||||
"rope_theta": 500000.0,
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.51.3",
|
||||
"use_cache": false,
|
||||
"vocab_size": 128256
|
||||
}
|
||||
762
evals/.hydra/config.yaml
Normal file
762
evals/.hydra/config.yaml
Normal file
@@ -0,0 +1,762 @@
|
||||
model:
|
||||
model_args:
|
||||
device_map: cuda
|
||||
pretrained_model_name_or_path: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
|
||||
attn_implementation: flash_attention_2
|
||||
torch_dtype: bfloat16
|
||||
model_handler: AutoModelForCausalLM
|
||||
tokenizer_args:
|
||||
pretrained_model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
|
||||
template_args:
|
||||
apply_chat_template: true
|
||||
system_prompt: You are a helpful assistant.
|
||||
system_prompt_with_special_tokens: '<|begin_of_text|><|start_header_id|>system<|end_header_id|>
|
||||
|
||||
|
||||
You are a helpful assistant.<|eot_id|>'
|
||||
user_start_tag: '<|start_header_id|>user<|end_header_id|>
|
||||
|
||||
|
||||
'
|
||||
user_end_tag: <|eot_id|>
|
||||
asst_start_tag: '<|start_header_id|>assistant<|end_header_id|>
|
||||
|
||||
|
||||
'
|
||||
asst_end_tag: <|eot_id|>
|
||||
date_string: 10 Apr 2025
|
||||
mode: eval
|
||||
task_name: tofu_Llama-3.1-8B-Instruct_forget05_NPO_lr2e-05_beta0.1_alpha2_epoch10
|
||||
seed: 0
|
||||
eval:
|
||||
tofu:
|
||||
metrics:
|
||||
forget_quality:
|
||||
pre_compute:
|
||||
forget_truth_ratio:
|
||||
pre_compute:
|
||||
forget_Q_A_PARA_Prob:
|
||||
datasets:
|
||||
TOFU_QA_forget_para:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: paraphrased_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
forget_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_forget_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: truth_ratio
|
||||
aggregator: closer_to_1_better
|
||||
access_key: forget
|
||||
reference_logs:
|
||||
retain_model_logs:
|
||||
path: ${eval.tofu.retain_logs_path}
|
||||
include:
|
||||
forget_truth_ratio:
|
||||
access_key: retain
|
||||
handler: ks_test
|
||||
model_utility:
|
||||
pre_compute:
|
||||
retain_Q_A_Prob:
|
||||
datasets:
|
||||
TOFU_QA_retain_eval:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: retain_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
retain_Q_A_ROUGE:
|
||||
datasets:
|
||||
TOFU_QA_retain_eval:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: retain_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
predict_with_generate: true
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: left
|
||||
index: index
|
||||
generation_args:
|
||||
do_sample: false
|
||||
top_p: null
|
||||
temperature: null
|
||||
max_new_tokens: 200
|
||||
use_cache: true
|
||||
handler: rouge
|
||||
rouge_type: rougeL_recall
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
retain_Truth_Ratio:
|
||||
pre_compute:
|
||||
retain_Q_A_PARA_Prob:
|
||||
datasets:
|
||||
TOFU_QA_retain_para:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: retain_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: paraphrased_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
retain_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_retain_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: retain_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: truth_ratio
|
||||
aggregator: true_better
|
||||
ra_Q_A_Prob_normalised:
|
||||
pre_compute:
|
||||
ra_Q_A_Prob:
|
||||
datasets:
|
||||
TOFU_QA_ra:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: real_authors_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
ra_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_ra_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: real_authors_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: probability_w_options
|
||||
ra_Q_A_ROUGE:
|
||||
datasets:
|
||||
TOFU_QA_ra:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: real_authors_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
predict_with_generate: true
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: left
|
||||
index: index
|
||||
generation_args:
|
||||
do_sample: false
|
||||
top_p: null
|
||||
temperature: null
|
||||
max_new_tokens: 200
|
||||
use_cache: true
|
||||
handler: rouge
|
||||
rouge_type: rougeL_recall
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
ra_Truth_Ratio:
|
||||
pre_compute:
|
||||
ra_Q_A_Prob:
|
||||
datasets:
|
||||
TOFU_QA_ra:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: real_authors_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
ra_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_ra_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: real_authors_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: truth_ratio
|
||||
aggregator: true_better
|
||||
wf_Q_A_Prob_normalised:
|
||||
pre_compute:
|
||||
wf_Q_A_Prob:
|
||||
datasets:
|
||||
TOFU_QA_wf:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: world_facts_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
wf_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_wf_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: world_facts_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: probability_w_options
|
||||
wf_Q_A_ROUGE:
|
||||
datasets:
|
||||
TOFU_QA_wf:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: world_facts_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
predict_with_generate: true
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: left
|
||||
index: index
|
||||
generation_args:
|
||||
do_sample: false
|
||||
top_p: null
|
||||
temperature: null
|
||||
max_new_tokens: 200
|
||||
use_cache: true
|
||||
handler: rouge
|
||||
rouge_type: rougeL_recall
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
wf_Truth_Ratio:
|
||||
pre_compute:
|
||||
wf_Q_A_Prob:
|
||||
datasets:
|
||||
TOFU_QA_wf:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: world_facts_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
wf_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_wf_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: world_facts_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: question
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: truth_ratio
|
||||
aggregator: true_better
|
||||
handler: hm_aggregate
|
||||
exact_memorization:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: exact_memorization
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
extraction_strength:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: extraction_strength
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
forget_Q_A_PARA_Prob:
|
||||
datasets:
|
||||
TOFU_QA_forget_para:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: paraphrased_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
forget_truth_ratio:
|
||||
pre_compute:
|
||||
forget_Q_A_PARA_Prob:
|
||||
datasets:
|
||||
TOFU_QA_forget_para:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: paraphrased_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: correct
|
||||
forget_Q_A_PERT_Prob:
|
||||
datasets:
|
||||
TOFU_QA_forget_pert:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: perturbed_answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
handler: probability
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: wrong
|
||||
handler: truth_ratio
|
||||
aggregator: closer_to_1_better
|
||||
forget_Q_A_gibberish:
|
||||
pre_compute:
|
||||
forget_Q_A_ROUGE:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
predict_with_generate: true
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: left
|
||||
index: index
|
||||
generation_args:
|
||||
do_sample: false
|
||||
top_p: null
|
||||
temperature: null
|
||||
max_new_tokens: 200
|
||||
use_cache: true
|
||||
handler: rouge
|
||||
rouge_type: rougeL_recall
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
access_key: text
|
||||
handler: classifier_prob
|
||||
batch_size: 32
|
||||
max_length: 32
|
||||
class_id: 0
|
||||
text_key: generation
|
||||
device: cuda
|
||||
classifier_model_args:
|
||||
pretrained_model_name_or_path: madhurjindal/autonlp-Gibberish-Detector-492513457
|
||||
classifier_tokenization_args:
|
||||
pretrained_model_name_or_path: madhurjindal/autonlp-Gibberish-Detector-492513457
|
||||
privleak:
|
||||
pre_compute:
|
||||
mia_min_k:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
access_key: forget
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
TOFU_QA_holdout:
|
||||
access_key: holdout
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.holdout_split}
|
||||
path: locuslab/TOFU
|
||||
split: train
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
handler: mia_min_k
|
||||
k: 0.4
|
||||
access_key: forget
|
||||
reference_logs:
|
||||
retain_model_logs:
|
||||
path: ${eval.tofu.retain_logs_path}
|
||||
include:
|
||||
mia_min_k:
|
||||
access_key: retain
|
||||
handler: privleak
|
||||
ref_value: 0.5
|
||||
mia_min_k_plus_plus:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
access_key: forget
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
TOFU_QA_holdout:
|
||||
access_key: holdout
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.holdout_split}
|
||||
path: locuslab/TOFU
|
||||
split: train
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
k: 0.4
|
||||
handler: mia_min_k_plus_plus
|
||||
mia_min_k:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
access_key: forget
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
TOFU_QA_holdout:
|
||||
access_key: holdout
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.holdout_split}
|
||||
path: locuslab/TOFU
|
||||
split: train
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
handler: mia_min_k
|
||||
k: 0.4
|
||||
mia_loss:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
access_key: forget
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
TOFU_QA_holdout:
|
||||
access_key: holdout
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.holdout_split}
|
||||
path: locuslab/TOFU
|
||||
split: train
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
handler: mia_loss
|
||||
mia_zlib:
|
||||
datasets:
|
||||
TOFU_QA_forget:
|
||||
access_key: forget
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.forget_split}_perturbed
|
||||
split: train
|
||||
path: locuslab/TOFU
|
||||
question_key: ${eval.tofu.question_key}
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
TOFU_QA_holdout:
|
||||
access_key: holdout
|
||||
handler: QADataset
|
||||
args:
|
||||
hf_args:
|
||||
name: ${eval.tofu.holdout_split}
|
||||
path: locuslab/TOFU
|
||||
split: train
|
||||
question_key: question
|
||||
answer_key: answer
|
||||
max_length: 512
|
||||
collators:
|
||||
DataCollatorForSupervisedDataset:
|
||||
handler: DataCollatorForSupervisedDataset
|
||||
args:
|
||||
padding_side: right
|
||||
index: index
|
||||
batch_size: ${eval.tofu.batch_size}
|
||||
handler: mia_zlib
|
||||
handler: TOFUEvaluator
|
||||
output_dir: ${paths.output_dir}
|
||||
overwrite: false
|
||||
forget_split: ${forget_split}
|
||||
holdout_split: ${holdout_split}
|
||||
retain_logs_path: ${retain_logs_path}
|
||||
question_key: question
|
||||
batch_size: 32
|
||||
paths:
|
||||
root_dir: .
|
||||
data_dir: ${paths.root_dir}/data/
|
||||
datasets: ${paths.root_dir}/configs/data/datasets
|
||||
output_dir: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals
|
||||
work_dir: ${hydra:runtime.cwd}
|
||||
forget_split: forget05
|
||||
holdout_split: holdout05
|
||||
retain_logs_path: saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
|
||||
288
evals/.hydra/hydra.yaml
Normal file
288
evals/.hydra/hydra.yaml
Normal file
@@ -0,0 +1,288 @@
|
||||
hydra:
|
||||
run:
|
||||
dir: ${paths.output_dir}
|
||||
sweep:
|
||||
dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
|
||||
subdir: ${hydra.job.num}
|
||||
launcher:
|
||||
_target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
|
||||
sweeper:
|
||||
_target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
|
||||
max_batch_size: null
|
||||
params: null
|
||||
help:
|
||||
app_name: ${hydra.job.name}
|
||||
header: '${hydra.help.app_name} is powered by Hydra.
|
||||
|
||||
'
|
||||
footer: 'Powered by Hydra (https://hydra.cc)
|
||||
|
||||
Use --hydra-help to view Hydra specific help
|
||||
|
||||
'
|
||||
template: '${hydra.help.header}
|
||||
|
||||
== Configuration groups ==
|
||||
|
||||
Compose your configuration from those groups (group=option)
|
||||
|
||||
|
||||
$APP_CONFIG_GROUPS
|
||||
|
||||
|
||||
== Config ==
|
||||
|
||||
Override anything in the config (foo.bar=value)
|
||||
|
||||
|
||||
$CONFIG
|
||||
|
||||
|
||||
${hydra.help.footer}
|
||||
|
||||
'
|
||||
hydra_help:
|
||||
template: 'Hydra (${hydra.runtime.version})
|
||||
|
||||
See https://hydra.cc for more info.
|
||||
|
||||
|
||||
== Flags ==
|
||||
|
||||
$FLAGS_HELP
|
||||
|
||||
|
||||
== Configuration groups ==
|
||||
|
||||
Compose your configuration from those groups (For example, append hydra/job_logging=disabled
|
||||
to command line)
|
||||
|
||||
|
||||
$HYDRA_CONFIG_GROUPS
|
||||
|
||||
|
||||
Use ''--cfg hydra'' to Show the Hydra config.
|
||||
|
||||
'
|
||||
hydra_help: ???
|
||||
hydra_logging:
|
||||
version: 1
|
||||
formatters:
|
||||
colorlog:
|
||||
(): colorlog.ColoredFormatter
|
||||
format: '[%(cyan)s%(asctime)s%(reset)s][%(purple)sHYDRA%(reset)s] %(message)s'
|
||||
handlers:
|
||||
console:
|
||||
class: logging.StreamHandler
|
||||
formatter: colorlog
|
||||
stream: ext://sys.stdout
|
||||
root:
|
||||
level: INFO
|
||||
handlers:
|
||||
- console
|
||||
disable_existing_loggers: false
|
||||
job_logging:
|
||||
version: 1
|
||||
formatters:
|
||||
simple:
|
||||
format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
|
||||
colorlog:
|
||||
(): colorlog.ColoredFormatter
|
||||
format: '[%(cyan)s%(asctime)s%(reset)s][%(blue)s%(name)s%(reset)s][%(log_color)s%(levelname)s%(reset)s]
|
||||
- %(message)s'
|
||||
log_colors:
|
||||
DEBUG: purple
|
||||
INFO: green
|
||||
WARNING: yellow
|
||||
ERROR: red
|
||||
CRITICAL: red
|
||||
handlers:
|
||||
console:
|
||||
class: logging.StreamHandler
|
||||
formatter: colorlog
|
||||
stream: ext://sys.stdout
|
||||
file:
|
||||
class: logging.FileHandler
|
||||
formatter: simple
|
||||
filename: ${hydra.runtime.output_dir}/eval.log
|
||||
root:
|
||||
level: INFO
|
||||
handlers:
|
||||
- console
|
||||
- file
|
||||
disable_existing_loggers: false
|
||||
env: {}
|
||||
mode: RUN
|
||||
searchpath: []
|
||||
callbacks: {}
|
||||
output_subdir: .hydra
|
||||
overrides:
|
||||
hydra:
|
||||
- hydra.mode=RUN
|
||||
task:
|
||||
- experiment=eval/tofu/default.yaml
|
||||
- forget_split=forget05
|
||||
- holdout_split=holdout05
|
||||
- model=Llama-3.1-8B-Instruct
|
||||
- task_name=tofu_Llama-3.1-8B-Instruct_forget05_NPO_lr2e-05_beta0.1_alpha2_epoch10
|
||||
- model.model_args.pretrained_model_name_or_path=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
|
||||
- paths.output_dir=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals
|
||||
- retain_logs_path=saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
|
||||
job:
|
||||
name: eval
|
||||
chdir: null
|
||||
override_dirname: experiment=eval/tofu/default.yaml,forget_split=forget05,holdout_split=holdout05,model.model_args.pretrained_model_name_or_path=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05,model=Llama-3.1-8B-Instruct,paths.output_dir=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals,retain_logs_path=saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json,task_name=tofu_Llama-3.1-8B-Instruct_forget05_NPO_lr2e-05_beta0.1_alpha2_epoch10
|
||||
id: ???
|
||||
num: ???
|
||||
config_name: eval.yaml
|
||||
env_set: {}
|
||||
env_copy: []
|
||||
config:
|
||||
override_dirname:
|
||||
kv_sep: '='
|
||||
item_sep: ','
|
||||
exclude_keys: []
|
||||
runtime:
|
||||
version: 1.3.0
|
||||
version_base: '1.3'
|
||||
cwd: /home/joaoabitante/speculative-decoding-unlearning
|
||||
config_sources:
|
||||
- path: hydra.conf
|
||||
schema: pkg
|
||||
provider: hydra
|
||||
- path: /home/joaoabitante/speculative-decoding-unlearning/configs
|
||||
schema: file
|
||||
provider: main
|
||||
- path: hydra_plugins.hydra_colorlog.conf
|
||||
schema: pkg
|
||||
provider: hydra-colorlog
|
||||
- path: ''
|
||||
schema: structured
|
||||
provider: schema
|
||||
output_dir: /home/joaoabitante/speculative-decoding-unlearning/saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals
|
||||
choices:
|
||||
adapter: null
|
||||
quantization: null
|
||||
experiment: eval/tofu/default.yaml
|
||||
hydra: eval
|
||||
paths: default
|
||||
eval: tofu
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.mia_zlib.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.mia_zlib.datasets: TOFU_MIA
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.mia_loss.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.mia_loss.datasets: TOFU_MIA
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.mia_min_k.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.mia_min_k.datasets: TOFU_MIA
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.mia_min_k_plus_plus.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.mia_min_k_plus_plus.datasets: TOFU_MIA
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.privleak.pre_compute.mia_min_k: mia_min_k
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.privleak.pre_compute.mia_min_k.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.privleak.pre_compute.mia_min_k.datasets: TOFU_MIA
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.forget_Q_A_gibberish.pre_compute.forget_Q_A_ROUGE: forget_Q_A_ROUGE
|
||||
eval/tofu_metrics/./../../generation@eval.tofu.metrics.forget_Q_A_gibberish.pre_compute.forget_Q_A_ROUGE.generation_args: default
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_Q_A_gibberish.pre_compute.forget_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_Q_A_gibberish.pre_compute.forget_Q_A_ROUGE.datasets: TOFU_QA_forget
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets: TOFU_QA_forget_pert
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets: TOFU_QA_forget_para
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_PARA_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_PARA_Prob.datasets: TOFU_QA_forget_para
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.extraction_strength.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.extraction_strength.datasets: TOFU_QA_forget
|
||||
eval/tofu_metrics/../../collator@eval.tofu.metrics.exact_memorization.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.exact_memorization.datasets: TOFU_QA_forget
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio: wf_Truth_Ratio
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.datasets
|
||||
: TOFU_QA_wf_pert
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.datasets
|
||||
: TOFU_QA_wf
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE: wf_Q_A_ROUGE
|
||||
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.generation_args: default
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.datasets: TOFU_QA_wf
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised: wf_Q_A_Prob_normalised
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.datasets
|
||||
: TOFU_QA_wf_pert
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.datasets
|
||||
: TOFU_QA_wf
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio: ra_Truth_Ratio
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.datasets
|
||||
: TOFU_QA_ra_pert
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.datasets
|
||||
: TOFU_QA_ra
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE: ra_Q_A_ROUGE
|
||||
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.generation_args: default
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.datasets: TOFU_QA_ra
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised: ra_Q_A_Prob_normalised
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.datasets
|
||||
: TOFU_QA_ra_pert
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.datasets
|
||||
: TOFU_QA_ra
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio: retain_Truth_Ratio
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob: retain_Q_A_PERT_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.datasets
|
||||
: TOFU_QA_retain_pert
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob: retain_Q_A_PARA_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.datasets
|
||||
: TOFU_QA_retain_para
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE: retain_Q_A_ROUGE
|
||||
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.generation_args: default
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.datasets: TOFU_QA_retain_eval
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob: retain_Q_A_Prob
|
||||
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.datasets: TOFU_QA_retain_eval
|
||||
eval/tofu_metrics/.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio: forget_Truth_Ratio
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets
|
||||
: TOFU_QA_forget_pert
|
||||
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
|
||||
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators
|
||||
: DataCollatorForSupervisedDatasetwithIndex
|
||||
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets
|
||||
: TOFU_QA_forget_para
|
||||
model: Llama-3.1-8B-Instruct
|
||||
hydra/env: default
|
||||
hydra/callbacks: null
|
||||
hydra/job_logging: colorlog
|
||||
hydra/hydra_logging: colorlog
|
||||
hydra/hydra_help: default
|
||||
hydra/help: default
|
||||
hydra/sweeper: basic
|
||||
hydra/launcher: basic
|
||||
hydra/output: default
|
||||
verbose: false
|
||||
8
evals/.hydra/overrides.yaml
Normal file
8
evals/.hydra/overrides.yaml
Normal file
@@ -0,0 +1,8 @@
|
||||
- experiment=eval/tofu/default.yaml
|
||||
- forget_split=forget05
|
||||
- holdout_split=holdout05
|
||||
- model=Llama-3.1-8B-Instruct
|
||||
- task_name=tofu_Llama-3.1-8B-Instruct_forget05_NPO_lr2e-05_beta0.1_alpha2_epoch10
|
||||
- model.model_args.pretrained_model_name_or_path=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
|
||||
- paths.output_dir=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals
|
||||
- retain_logs_path=saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
|
||||
33687
evals/TOFU_EVAL.json
Normal file
33687
evals/TOFU_EVAL.json
Normal file
File diff suppressed because it is too large
Load Diff
14
evals/TOFU_SUMMARY.json
Normal file
14
evals/TOFU_SUMMARY.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"exact_memorization": 0.5072669323906303,
|
||||
"extraction_strength": 0.05579637056005085,
|
||||
"forget_Q_A_PARA_Prob": 0.023710637748226873,
|
||||
"forget_Q_A_gibberish": 0.8972614184241684,
|
||||
"forget_quality": 0.011843449760085422,
|
||||
"forget_truth_ratio": 0.6632387214939777,
|
||||
"mia_loss": 0.06501250000000001,
|
||||
"mia_min_k": 0.06733750000000001,
|
||||
"mia_min_k_plus_plus": 0.120575,
|
||||
"mia_zlib": 0.10685,
|
||||
"model_utility": 0.5678715338202777,
|
||||
"privleak": 44.8683597320801
|
||||
}
|
||||
59
evals/eval.log
Normal file
59
evals/eval.log
Normal file
@@ -0,0 +1,59 @@
|
||||
[2026-08-24 16:02:28,521][model][INFO] - Setting pad_token as eos token: <|eot_id|>
|
||||
[2026-08-24 16:02:28,525][evaluator][INFO] - Evaluations stored in the experiment directory: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals
|
||||
[2026-08-24 16:02:28,527][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||
[2026-08-24 16:02:28,527][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals/TOFU_EVAL.json
|
||||
[2026-08-24 16:02:28,527][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals/TOFU_SUMMARY.json
|
||||
[2026-08-24 16:02:30,471][metrics][INFO] - Loading evaluations from saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
|
||||
[2026-08-24 16:02:30,481][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||
[2026-08-24 16:02:37,358][metrics][INFO] - Loading evaluations from saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
|
||||
[2026-08-24 16:02:37,368][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||
[2026-08-24 16:03:02,886][metrics][INFO] - Loading evaluations from saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
|
||||
[2026-08-24 16:03:02,896][metrics][INFO] - Evaluating forget_truth_ratio
|
||||
[2026-08-24 16:03:02,897][metrics][INFO] - Loading evaluations from saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
|
||||
[2026-08-24 16:03:02,905][metrics][INFO] - Evaluating forget_quality
|
||||
[2026-08-24 16:03:02,906][evaluator][INFO] - Result for metric forget_quality: 0.011843449760085422
|
||||
[2026-08-24 16:03:04,138][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||
[2026-08-24 16:03:14,659][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||
[2026-08-24 16:04:00,406][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||
[2026-08-24 16:04:11,558][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||
[2026-08-24 16:05:00,809][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||
[2026-08-24 16:05:02,042][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||
[2026-08-24 16:05:04,603][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||
[2026-08-24 16:05:09,445][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||
[2026-08-24 16:05:10,390][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||
[2026-08-24 16:05:18,593][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||
[2026-08-24 16:05:18,593][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||
[2026-08-24 16:05:18,593][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||
[2026-08-24 16:05:19,837][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||
[2026-08-24 16:05:22,437][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||
[2026-08-24 16:05:27,584][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||
[2026-08-24 16:05:28,434][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||
[2026-08-24 16:05:40,357][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||
[2026-08-24 16:05:40,357][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||
[2026-08-24 16:05:40,357][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||
[2026-08-24 16:05:40,357][metrics][INFO] - Evaluating model_utility
|
||||
[2026-08-24 16:05:40,358][evaluator][INFO] - Result for metric model_utility: 0.5678715338202777
|
||||
[2026-08-24 16:05:41,597][metrics][INFO] - Evaluating exact_memorization
|
||||
[2026-08-24 16:05:45,731][evaluator][INFO] - Result for metric exact_memorization: 0.5072669323906303
|
||||
[2026-08-24 16:05:46,739][metrics][INFO] - Evaluating extraction_strength
|
||||
[2026-08-24 16:05:51,044][evaluator][INFO] - Result for metric extraction_strength: 0.05579637056005085
|
||||
[2026-08-24 16:05:51,073][evaluator][INFO] - Skipping forget_Q_A_PARA_Prob, already evaluated.
|
||||
[2026-08-24 16:05:51,074][evaluator][INFO] - Result for metric forget_Q_A_PARA_Prob: 0.023710637748226873
|
||||
[2026-08-24 16:05:51,074][evaluator][INFO] - Skipping forget_truth_ratio, already evaluated.
|
||||
[2026-08-24 16:05:51,074][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6632387214939777
|
||||
[2026-08-24 16:05:51,977][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||
[2026-08-24 16:06:21,886][metrics][INFO] - Evaluating forget_Q_A_gibberish
|
||||
[2026-08-24 16:06:22,437][evaluator][INFO] - Result for metric forget_Q_A_gibberish: 0.8972614184241684
|
||||
[2026-08-24 16:06:24,641][metrics][INFO] - Loading evaluations from saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
|
||||
[2026-08-24 16:06:24,651][metrics][INFO] - Evaluating mia_min_k
|
||||
[2026-08-24 16:06:33,158][metrics][INFO] - Loading evaluations from saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
|
||||
[2026-08-24 16:06:33,167][metrics][INFO] - Evaluating privleak
|
||||
[2026-08-24 16:06:33,167][evaluator][INFO] - Result for metric privleak: 44.8683597320801
|
||||
[2026-08-24 16:06:35,337][metrics][INFO] - Evaluating mia_min_k_plus_plus
|
||||
[2026-08-24 16:06:52,206][evaluator][INFO] - Result for metric mia_min_k_plus_plus: 0.120575
|
||||
[2026-08-24 16:06:52,242][evaluator][INFO] - Skipping mia_min_k, already evaluated.
|
||||
[2026-08-24 16:06:52,242][evaluator][INFO] - Result for metric mia_min_k: 0.06733750000000001
|
||||
[2026-08-24 16:06:54,534][metrics][INFO] - Evaluating mia_loss
|
||||
[2026-08-24 16:07:04,898][evaluator][INFO] - Result for metric mia_loss: 0.06501250000000001
|
||||
[2026-08-24 16:07:07,081][metrics][INFO] - Evaluating mia_zlib
|
||||
[2026-08-24 16:07:17,516][evaluator][INFO] - Result for metric mia_zlib: 0.10685
|
||||
8
evals/profiling.json
Normal file
8
evals/profiling.json
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"timings_sec": {
|
||||
"evaluation": 289.029
|
||||
},
|
||||
"vram_peak_mb": {
|
||||
"evaluation": 23911.0
|
||||
}
|
||||
}
|
||||
12
generation_config.json
Normal file
12
generation_config.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"bos_token_id": 128000,
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
128001,
|
||||
128008,
|
||||
128009
|
||||
],
|
||||
"temperature": 0.6,
|
||||
"top_p": 0.9,
|
||||
"transformers_version": "4.51.3"
|
||||
}
|
||||
3
model-00001-of-00004.safetensors
Normal file
3
model-00001-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1e06a2732a84e110b067dc1ec93371ec38518858b9d2e20d2410681cb04fa812
|
||||
size 4976698672
|
||||
3
model-00002-of-00004.safetensors
Normal file
3
model-00002-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a35868b7d545ab60125ee04af52f42d2608a0c0a83be93b13af08edc4a8e09d6
|
||||
size 4999802720
|
||||
3
model-00003-of-00004.safetensors
Normal file
3
model-00003-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:daf94e1811490f567f1e7f1cd1f00a669fae76b0a6aa610f03e23eed69275410
|
||||
size 4915916176
|
||||
3
model-00004-of-00004.safetensors
Normal file
3
model-00004-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:afd75d205a0a290791a0a4b8d6a93b02ee47fd9c2318bc599648fb971ee832e8
|
||||
size 1168138808
|
||||
298
model.safetensors.index.json
Normal file
298
model.safetensors.index.json
Normal file
@@ -0,0 +1,298 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_size": 16060522496
|
||||
},
|
||||
"weight_map": {
|
||||
"lm_head.weight": "model-00004-of-00004.safetensors",
|
||||
"model.embed_tokens.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.norm.weight": "model-00004-of-00004.safetensors"
|
||||
}
|
||||
}
|
||||
8
profiling.json
Normal file
8
profiling.json
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"timings_sec": {
|
||||
"training": 3274.995
|
||||
},
|
||||
"vram_peak_mb": {
|
||||
"training": 46807.7
|
||||
}
|
||||
}
|
||||
17
special_tokens_map.json
Normal file
17
special_tokens_map.json
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<|begin_of_text|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|eot_id|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": "<|eot_id|>"
|
||||
}
|
||||
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
Binary file not shown.
2064
tokenizer_config.json
Normal file
2064
tokenizer_config.json
Normal file
File diff suppressed because it is too large
Load Diff
127
trainer_state.json
Normal file
127
trainer_state.json
Normal file
@@ -0,0 +1,127 @@
|
||||
{
|
||||
"best_global_step": null,
|
||||
"best_metric": null,
|
||||
"best_model_checkpoint": null,
|
||||
"epoch": 8.64,
|
||||
"eval_steps": 500,
|
||||
"global_step": 60,
|
||||
"is_hyper_param_search": false,
|
||||
"is_local_process_zero": true,
|
||||
"is_world_process_zero": true,
|
||||
"log_history": [
|
||||
{
|
||||
"epoch": 0.8,
|
||||
"grad_norm": 334.85599352203303,
|
||||
"learning_rate": 1.3333333333333333e-05,
|
||||
"loss": 54.718,
|
||||
"step": 5
|
||||
},
|
||||
{
|
||||
"epoch": 1.48,
|
||||
"grad_norm": 202.7554645505365,
|
||||
"learning_rate": 1.888888888888889e-05,
|
||||
"loss": 17.2721,
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"epoch": 2.16,
|
||||
"grad_norm": 80.19389428909676,
|
||||
"learning_rate": 1.7037037037037038e-05,
|
||||
"loss": 9.8854,
|
||||
"step": 15
|
||||
},
|
||||
{
|
||||
"epoch": 2.96,
|
||||
"grad_norm": 70.15487433345591,
|
||||
"learning_rate": 1.5185185185185187e-05,
|
||||
"loss": 9.1224,
|
||||
"step": 20
|
||||
},
|
||||
{
|
||||
"epoch": 3.64,
|
||||
"grad_norm": 67.74738211665279,
|
||||
"learning_rate": 1.3333333333333333e-05,
|
||||
"loss": 6.7069,
|
||||
"step": 25
|
||||
},
|
||||
{
|
||||
"epoch": 4.32,
|
||||
"grad_norm": 59.21687170152334,
|
||||
"learning_rate": 1.1481481481481482e-05,
|
||||
"loss": 5.8678,
|
||||
"step": 30
|
||||
},
|
||||
{
|
||||
"epoch": 5.0,
|
||||
"grad_norm": 63.030732034440945,
|
||||
"learning_rate": 9.62962962962963e-06,
|
||||
"loss": 4.7996,
|
||||
"step": 35
|
||||
},
|
||||
{
|
||||
"epoch": 5.8,
|
||||
"grad_norm": 60.56173023691404,
|
||||
"learning_rate": 7.77777777777778e-06,
|
||||
"loss": 5.4373,
|
||||
"step": 40
|
||||
},
|
||||
{
|
||||
"epoch": 6.48,
|
||||
"grad_norm": 59.13232869822844,
|
||||
"learning_rate": 5.925925925925926e-06,
|
||||
"loss": 4.6898,
|
||||
"step": 45
|
||||
},
|
||||
{
|
||||
"epoch": 7.16,
|
||||
"grad_norm": 56.71805846317398,
|
||||
"learning_rate": 4.074074074074074e-06,
|
||||
"loss": 3.3341,
|
||||
"step": 50
|
||||
},
|
||||
{
|
||||
"epoch": 7.96,
|
||||
"grad_norm": 76.99880625336664,
|
||||
"learning_rate": 2.222222222222222e-06,
|
||||
"loss": 4.3757,
|
||||
"step": 55
|
||||
},
|
||||
{
|
||||
"epoch": 8.64,
|
||||
"grad_norm": 63.78644421102578,
|
||||
"learning_rate": 3.7037037037037036e-07,
|
||||
"loss": 3.545,
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"epoch": 8.64,
|
||||
"step": 60,
|
||||
"total_flos": 0.0,
|
||||
"train_loss": 10.812849680582682,
|
||||
"train_runtime": 3254.027,
|
||||
"train_samples_per_second": 0.615,
|
||||
"train_steps_per_second": 0.018
|
||||
}
|
||||
],
|
||||
"logging_steps": 5,
|
||||
"max_steps": 60,
|
||||
"num_input_tokens_seen": 0,
|
||||
"num_train_epochs": 10,
|
||||
"save_steps": 500,
|
||||
"stateful_callbacks": {
|
||||
"TrainerControl": {
|
||||
"args": {
|
||||
"should_epoch_stop": false,
|
||||
"should_evaluate": false,
|
||||
"should_log": false,
|
||||
"should_save": false,
|
||||
"should_training_stop": false
|
||||
},
|
||||
"attributes": {}
|
||||
}
|
||||
},
|
||||
"total_flos": 0.0,
|
||||
"train_batch_size": 4,
|
||||
"trial_name": null,
|
||||
"trial_params": null
|
||||
}
|
||||
Reference in New Issue
Block a user