JoaoBoer/tofu_Llama-3.1-8B-Instruct_forget05_NPO
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: JoaoBoer/tofu_Llama-3.1-8B-Instruct_forget05_NPO
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-09-28 03:45:18 +08:00
commit d5c22b118a
25 changed files with 38661 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

762
evals/.hydra/config.yaml Normal file
View File

@@ -0,0 +1,762 @@
model:
model_args:
device_map: cuda
pretrained_model_name_or_path: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
attn_implementation: flash_attention_2
torch_dtype: bfloat16
model_handler: AutoModelForCausalLM
tokenizer_args:
pretrained_model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
template_args:
apply_chat_template: true
system_prompt: You are a helpful assistant.
system_prompt_with_special_tokens: '<|begin_of_text|><|start_header_id|>system<|end_header_id|>
You are a helpful assistant.<|eot_id|>'
user_start_tag: '<|start_header_id|>user<|end_header_id|>
'
user_end_tag: <|eot_id|>
asst_start_tag: '<|start_header_id|>assistant<|end_header_id|>
'
asst_end_tag: <|eot_id|>
date_string: 10 Apr 2025
mode: eval
task_name: tofu_Llama-3.1-8B-Instruct_forget05_NPO_lr2e-05_beta0.1_alpha2_epoch10
seed: 0
eval:
tofu:
metrics:
forget_quality:
pre_compute:
forget_truth_ratio:
pre_compute:
forget_Q_A_PARA_Prob:
datasets:
TOFU_QA_forget_para:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
forget_Q_A_PERT_Prob:
datasets:
TOFU_QA_forget_pert:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: closer_to_1_better
access_key: forget
reference_logs:
retain_model_logs:
path: ${eval.tofu.retain_logs_path}
include:
forget_truth_ratio:
access_key: retain
handler: ks_test
model_utility:
pre_compute:
retain_Q_A_Prob:
datasets:
TOFU_QA_retain_eval:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
retain_Q_A_ROUGE:
datasets:
TOFU_QA_retain_eval:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
retain_Truth_Ratio:
pre_compute:
retain_Q_A_PARA_Prob:
datasets:
TOFU_QA_retain_para:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
retain_Q_A_PERT_Prob:
datasets:
TOFU_QA_retain_pert:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: true_better
ra_Q_A_Prob_normalised:
pre_compute:
ra_Q_A_Prob:
datasets:
TOFU_QA_ra:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
ra_Q_A_PERT_Prob:
datasets:
TOFU_QA_ra_pert:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: probability_w_options
ra_Q_A_ROUGE:
datasets:
TOFU_QA_ra:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
ra_Truth_Ratio:
pre_compute:
ra_Q_A_Prob:
datasets:
TOFU_QA_ra:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
ra_Q_A_PERT_Prob:
datasets:
TOFU_QA_ra_pert:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: true_better
wf_Q_A_Prob_normalised:
pre_compute:
wf_Q_A_Prob:
datasets:
TOFU_QA_wf:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
wf_Q_A_PERT_Prob:
datasets:
TOFU_QA_wf_pert:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: probability_w_options
wf_Q_A_ROUGE:
datasets:
TOFU_QA_wf:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
wf_Truth_Ratio:
pre_compute:
wf_Q_A_Prob:
datasets:
TOFU_QA_wf:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
wf_Q_A_PERT_Prob:
datasets:
TOFU_QA_wf_pert:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: true_better
handler: hm_aggregate
exact_memorization:
datasets:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: exact_memorization
batch_size: ${eval.tofu.batch_size}
extraction_strength:
datasets:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: extraction_strength
batch_size: ${eval.tofu.batch_size}
forget_Q_A_PARA_Prob:
datasets:
TOFU_QA_forget_para:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
forget_truth_ratio:
pre_compute:
forget_Q_A_PARA_Prob:
datasets:
TOFU_QA_forget_para:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
forget_Q_A_PERT_Prob:
datasets:
TOFU_QA_forget_pert:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: closer_to_1_better
forget_Q_A_gibberish:
pre_compute:
forget_Q_A_ROUGE:
datasets:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
access_key: text
handler: classifier_prob
batch_size: 32
max_length: 32
class_id: 0
text_key: generation
device: cuda
classifier_model_args:
pretrained_model_name_or_path: madhurjindal/autonlp-Gibberish-Detector-492513457
classifier_tokenization_args:
pretrained_model_name_or_path: madhurjindal/autonlp-Gibberish-Detector-492513457
privleak:
pre_compute:
mia_min_k:
datasets:
TOFU_QA_forget:
access_key: forget
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
TOFU_QA_holdout:
access_key: holdout
handler: QADataset
args:
hf_args:
name: ${eval.tofu.holdout_split}
path: locuslab/TOFU
split: train
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
batch_size: ${eval.tofu.batch_size}
handler: mia_min_k
k: 0.4
access_key: forget
reference_logs:
retain_model_logs:
path: ${eval.tofu.retain_logs_path}
include:
mia_min_k:
access_key: retain
handler: privleak
ref_value: 0.5
mia_min_k_plus_plus:
datasets:
TOFU_QA_forget:
access_key: forget
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
TOFU_QA_holdout:
access_key: holdout
handler: QADataset
args:
hf_args:
name: ${eval.tofu.holdout_split}
path: locuslab/TOFU
split: train
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
batch_size: ${eval.tofu.batch_size}
k: 0.4
handler: mia_min_k_plus_plus
mia_min_k:
datasets:
TOFU_QA_forget:
access_key: forget
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
TOFU_QA_holdout:
access_key: holdout
handler: QADataset
args:
hf_args:
name: ${eval.tofu.holdout_split}
path: locuslab/TOFU
split: train
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
batch_size: ${eval.tofu.batch_size}
handler: mia_min_k
k: 0.4
mia_loss:
datasets:
TOFU_QA_forget:
access_key: forget
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
TOFU_QA_holdout:
access_key: holdout
handler: QADataset
args:
hf_args:
name: ${eval.tofu.holdout_split}
path: locuslab/TOFU
split: train
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
batch_size: ${eval.tofu.batch_size}
handler: mia_loss
mia_zlib:
datasets:
TOFU_QA_forget:
access_key: forget
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
TOFU_QA_holdout:
access_key: holdout
handler: QADataset
args:
hf_args:
name: ${eval.tofu.holdout_split}
path: locuslab/TOFU
split: train
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
batch_size: ${eval.tofu.batch_size}
handler: mia_zlib
handler: TOFUEvaluator
output_dir: ${paths.output_dir}
overwrite: false
forget_split: ${forget_split}
holdout_split: ${holdout_split}
retain_logs_path: ${retain_logs_path}
question_key: question
batch_size: 32
paths:
root_dir: .
data_dir: ${paths.root_dir}/data/
datasets: ${paths.root_dir}/configs/data/datasets
output_dir: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals
work_dir: ${hydra:runtime.cwd}
forget_split: forget05
holdout_split: holdout05
retain_logs_path: saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json

288
evals/.hydra/hydra.yaml Normal file
View File

@@ -0,0 +1,288 @@
hydra:
run:
dir: ${paths.output_dir}
sweep:
dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
subdir: ${hydra.job.num}
launcher:
_target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
sweeper:
_target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
max_batch_size: null
params: null
help:
app_name: ${hydra.job.name}
header: '${hydra.help.app_name} is powered by Hydra.
'
footer: 'Powered by Hydra (https://hydra.cc)
Use --hydra-help to view Hydra specific help
'
template: '${hydra.help.header}
== Configuration groups ==
Compose your configuration from those groups (group=option)
$APP_CONFIG_GROUPS
== Config ==
Override anything in the config (foo.bar=value)
$CONFIG
${hydra.help.footer}
'
hydra_help:
template: 'Hydra (${hydra.runtime.version})
See https://hydra.cc for more info.
== Flags ==
$FLAGS_HELP
== Configuration groups ==
Compose your configuration from those groups (For example, append hydra/job_logging=disabled
to command line)
$HYDRA_CONFIG_GROUPS
Use ''--cfg hydra'' to Show the Hydra config.
'
hydra_help: ???
hydra_logging:
version: 1
formatters:
colorlog:
(): colorlog.ColoredFormatter
format: '[%(cyan)s%(asctime)s%(reset)s][%(purple)sHYDRA%(reset)s] %(message)s'
handlers:
console:
class: logging.StreamHandler
formatter: colorlog
stream: ext://sys.stdout
root:
level: INFO
handlers:
- console
disable_existing_loggers: false
job_logging:
version: 1
formatters:
simple:
format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
colorlog:
(): colorlog.ColoredFormatter
format: '[%(cyan)s%(asctime)s%(reset)s][%(blue)s%(name)s%(reset)s][%(log_color)s%(levelname)s%(reset)s]
- %(message)s'
log_colors:
DEBUG: purple
INFO: green
WARNING: yellow
ERROR: red
CRITICAL: red
handlers:
console:
class: logging.StreamHandler
formatter: colorlog
stream: ext://sys.stdout
file:
class: logging.FileHandler
formatter: simple
filename: ${hydra.runtime.output_dir}/eval.log
root:
level: INFO
handlers:
- console
- file
disable_existing_loggers: false
env: {}
mode: RUN
searchpath: []
callbacks: {}
output_subdir: .hydra
overrides:
hydra:
- hydra.mode=RUN
task:
- experiment=eval/tofu/default.yaml
- forget_split=forget05
- holdout_split=holdout05
- model=Llama-3.1-8B-Instruct
- task_name=tofu_Llama-3.1-8B-Instruct_forget05_NPO_lr2e-05_beta0.1_alpha2_epoch10
- model.model_args.pretrained_model_name_or_path=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
- paths.output_dir=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals
- retain_logs_path=saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
job:
name: eval
chdir: null
override_dirname: experiment=eval/tofu/default.yaml,forget_split=forget05,holdout_split=holdout05,model.model_args.pretrained_model_name_or_path=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05,model=Llama-3.1-8B-Instruct,paths.output_dir=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals,retain_logs_path=saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json,task_name=tofu_Llama-3.1-8B-Instruct_forget05_NPO_lr2e-05_beta0.1_alpha2_epoch10
id: ???
num: ???
config_name: eval.yaml
env_set: {}
env_copy: []
config:
override_dirname:
kv_sep: '='
item_sep: ','
exclude_keys: []
runtime:
version: 1.3.0
version_base: '1.3'
cwd: /home/joaoabitante/speculative-decoding-unlearning
config_sources:
- path: hydra.conf
schema: pkg
provider: hydra
- path: /home/joaoabitante/speculative-decoding-unlearning/configs
schema: file
provider: main
- path: hydra_plugins.hydra_colorlog.conf
schema: pkg
provider: hydra-colorlog
- path: ''
schema: structured
provider: schema
output_dir: /home/joaoabitante/speculative-decoding-unlearning/saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals
choices:
adapter: null
quantization: null
experiment: eval/tofu/default.yaml
hydra: eval
paths: default
eval: tofu
eval/tofu_metrics/../../collator@eval.tofu.metrics.mia_zlib.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.mia_zlib.datasets: TOFU_MIA
eval/tofu_metrics/../../collator@eval.tofu.metrics.mia_loss.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.mia_loss.datasets: TOFU_MIA
eval/tofu_metrics/../../collator@eval.tofu.metrics.mia_min_k.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.mia_min_k.datasets: TOFU_MIA
eval/tofu_metrics/../../collator@eval.tofu.metrics.mia_min_k_plus_plus.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.mia_min_k_plus_plus.datasets: TOFU_MIA
eval/tofu_metrics/.@eval.tofu.metrics.privleak.pre_compute.mia_min_k: mia_min_k
eval/tofu_metrics/./../../collator@eval.tofu.metrics.privleak.pre_compute.mia_min_k.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.privleak.pre_compute.mia_min_k.datasets: TOFU_MIA
eval/tofu_metrics/.@eval.tofu.metrics.forget_Q_A_gibberish.pre_compute.forget_Q_A_ROUGE: forget_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.forget_Q_A_gibberish.pre_compute.forget_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_Q_A_gibberish.pre_compute.forget_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_Q_A_gibberish.pre_compute.forget_Q_A_ROUGE.datasets: TOFU_QA_forget
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets: TOFU_QA_forget_pert
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets: TOFU_QA_forget_para
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_PARA_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_PARA_Prob.datasets: TOFU_QA_forget_para
eval/tofu_metrics/../../collator@eval.tofu.metrics.extraction_strength.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.extraction_strength.datasets: TOFU_QA_forget
eval/tofu_metrics/../../collator@eval.tofu.metrics.exact_memorization.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.exact_memorization.datasets: TOFU_QA_forget
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio: wf_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.datasets
: TOFU_QA_wf_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.datasets
: TOFU_QA_wf
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE: wf_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.datasets: TOFU_QA_wf
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised: wf_Q_A_Prob_normalised
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.datasets
: TOFU_QA_wf_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.datasets
: TOFU_QA_wf
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio: ra_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.datasets
: TOFU_QA_ra_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.datasets
: TOFU_QA_ra
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE: ra_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.datasets: TOFU_QA_ra
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised: ra_Q_A_Prob_normalised
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.datasets
: TOFU_QA_ra_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.datasets
: TOFU_QA_ra
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio: retain_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob: retain_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.datasets
: TOFU_QA_retain_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob: retain_Q_A_PARA_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.datasets
: TOFU_QA_retain_para
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE: retain_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.datasets: TOFU_QA_retain_eval
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob: retain_Q_A_Prob
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.datasets: TOFU_QA_retain_eval
eval/tofu_metrics/.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio: forget_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets
: TOFU_QA_forget_pert
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets
: TOFU_QA_forget_para
model: Llama-3.1-8B-Instruct
hydra/env: default
hydra/callbacks: null
hydra/job_logging: colorlog
hydra/hydra_logging: colorlog
hydra/hydra_help: default
hydra/help: default
hydra/sweeper: basic
hydra/launcher: basic
hydra/output: default
verbose: false

8
evals/.hydra/overrides.yaml Normal file
View File

@@ -0,0 +1,8 @@
- experiment=eval/tofu/default.yaml
- forget_split=forget05
- holdout_split=holdout05
- model=Llama-3.1-8B-Instruct
- task_name=tofu_Llama-3.1-8B-Instruct_forget05_NPO_lr2e-05_beta0.1_alpha2_epoch10
- model.model_args.pretrained_model_name_or_path=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05
- paths.output_dir=saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals
- retain_logs_path=saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json

33687
evals/TOFU_EVAL.json Normal file
View File

File diff suppressed because it is too large Load Diff

14
evals/TOFU_SUMMARY.json Normal file
View File

@@ -0,0 +1,14 @@
{
"exact_memorization": 0.5072669323906303,
"extraction_strength": 0.05579637056005085,
"forget_Q_A_PARA_Prob": 0.023710637748226873,
"forget_Q_A_gibberish": 0.8972614184241684,
"forget_quality": 0.011843449760085422,
"forget_truth_ratio": 0.6632387214939777,
"mia_loss": 0.06501250000000001,
"mia_min_k": 0.06733750000000001,
"mia_min_k_plus_plus": 0.120575,
"mia_zlib": 0.10685,
"model_utility": 0.5678715338202777,
"privleak": 44.8683597320801
}

59
evals/eval.log Normal file
View File

@@ -0,0 +1,59 @@
[2026-08-24 16:02:28,521][model][INFO] - Setting pad_token as eos token: <|eot_id|>
[2026-08-24 16:02:28,525][evaluator][INFO] - Evaluations stored in the experiment directory: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals
[2026-08-24 16:02:28,527][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-08-24 16:02:28,527][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals/TOFU_EVAL.json
[2026-08-24 16:02:28,527][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/baselines/tofu/NPO/Llama-3.1-8B-Instruct/forget05/evals/TOFU_SUMMARY.json
[2026-08-24 16:02:30,471][metrics][INFO] - Loading evaluations from saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
[2026-08-24 16:02:30,481][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-08-24 16:02:37,358][metrics][INFO] - Loading evaluations from saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
[2026-08-24 16:02:37,368][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-08-24 16:03:02,886][metrics][INFO] - Loading evaluations from saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
[2026-08-24 16:03:02,896][metrics][INFO] - Evaluating forget_truth_ratio
[2026-08-24 16:03:02,897][metrics][INFO] - Loading evaluations from saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
[2026-08-24 16:03:02,905][metrics][INFO] - Evaluating forget_quality
[2026-08-24 16:03:02,906][evaluator][INFO] - Result for metric forget_quality: 0.011843449760085422
[2026-08-24 16:03:04,138][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-08-24 16:03:14,659][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-08-24 16:04:00,406][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-08-24 16:04:11,558][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-08-24 16:05:00,809][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-08-24 16:05:02,042][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-08-24 16:05:04,603][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-08-24 16:05:09,445][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-08-24 16:05:10,390][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-08-24 16:05:18,593][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-08-24 16:05:18,593][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-08-24 16:05:18,593][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-08-24 16:05:19,837][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-08-24 16:05:22,437][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-08-24 16:05:27,584][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-08-24 16:05:28,434][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-08-24 16:05:40,357][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-08-24 16:05:40,357][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-08-24 16:05:40,357][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-08-24 16:05:40,357][metrics][INFO] - Evaluating model_utility
[2026-08-24 16:05:40,358][evaluator][INFO] - Result for metric model_utility: 0.5678715338202777
[2026-08-24 16:05:41,597][metrics][INFO] - Evaluating exact_memorization
[2026-08-24 16:05:45,731][evaluator][INFO] - Result for metric exact_memorization: 0.5072669323906303
[2026-08-24 16:05:46,739][metrics][INFO] - Evaluating extraction_strength
[2026-08-24 16:05:51,044][evaluator][INFO] - Result for metric extraction_strength: 0.05579637056005085
[2026-08-24 16:05:51,073][evaluator][INFO] - Skipping forget_Q_A_PARA_Prob, already evaluated.
[2026-08-24 16:05:51,074][evaluator][INFO] - Result for metric forget_Q_A_PARA_Prob: 0.023710637748226873
[2026-08-24 16:05:51,074][evaluator][INFO] - Skipping forget_truth_ratio, already evaluated.
[2026-08-24 16:05:51,074][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6632387214939777
[2026-08-24 16:05:51,977][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-08-24 16:06:21,886][metrics][INFO] - Evaluating forget_Q_A_gibberish
[2026-08-24 16:06:22,437][evaluator][INFO] - Result for metric forget_Q_A_gibberish: 0.8972614184241684
[2026-08-24 16:06:24,641][metrics][INFO] - Loading evaluations from saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
[2026-08-24 16:06:24,651][metrics][INFO] - Evaluating mia_min_k
[2026-08-24 16:06:33,158][metrics][INFO] - Loading evaluations from saves/eval/tofu_Llama-3.1-8B-Instruct_retain95/TOFU_EVAL.json
[2026-08-24 16:06:33,167][metrics][INFO] - Evaluating privleak
[2026-08-24 16:06:33,167][evaluator][INFO] - Result for metric privleak: 44.8683597320801
[2026-08-24 16:06:35,337][metrics][INFO] - Evaluating mia_min_k_plus_plus
[2026-08-24 16:06:52,206][evaluator][INFO] - Result for metric mia_min_k_plus_plus: 0.120575
[2026-08-24 16:06:52,242][evaluator][INFO] - Skipping mia_min_k, already evaluated.
[2026-08-24 16:06:52,242][evaluator][INFO] - Result for metric mia_min_k: 0.06733750000000001
[2026-08-24 16:06:54,534][metrics][INFO] - Evaluating mia_loss
[2026-08-24 16:07:04,898][evaluator][INFO] - Result for metric mia_loss: 0.06501250000000001
[2026-08-24 16:07:07,081][metrics][INFO] - Evaluating mia_zlib
[2026-08-24 16:07:17,516][evaluator][INFO] - Result for metric mia_zlib: 0.10685

8
evals/profiling.json Normal file
View File

@@ -0,0 +1,8 @@
{
"timings_sec": {
"evaluation": 289.029
},
"vram_peak_mb": {
"evaluation": 23911.0
}
}