Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-06-25 16:56:29 +08:00
commit 987a56dfe0
15 changed files with 4179 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.gitattributes vendored Normal file
View File

@@ -0,0 +1,36 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text

663
.hydra/config.yaml Normal file
View File

@@ -0,0 +1,663 @@
model:
model_args:
pretrained_model_name_or_path: open-unlearning/tofu_Llama-3.2-1B-Instruct_full
attn_implementation: sdpa
torch_dtype: bfloat16
tokenizer_args:
pretrained_model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
template_args:
apply_chat_template: true
system_prompt: You are a helpful assistant.
system_prompt_with_special_tokens: '<|begin_of_text|><|start_header_id|>system<|end_header_id|>
You are a helpful assistant.<|eot_id|>'
user_start_tag: '<|start_header_id|>user<|end_header_id|>
'
user_end_tag: <|eot_id|>
asst_start_tag: '<|start_header_id|>assistant<|end_header_id|>
'
asst_end_tag: <|eot_id|>
date_string: 10 Apr 2025
trainer:
handler: NPO
args:
per_device_train_batch_size: 4
per_device_eval_batch_size: 16
gradient_accumulation_steps: 4
learning_rate: 1.0e-05
bf16: true
bf16_full_eval: true
logging_steps: 5
output_dir: ${paths.output_dir}
logging_dir: ${trainer.args.output_dir}/logs
report_to: tensorboard
ddp_find_unused_parameters: true
gradient_checkpointing: false
optim: paged_adamw_32bit
save_strategy: 'no'
save_only_model: true
weight_decay: 0.01
do_train: true
do_eval: false
eval_on_start: true
eval_strategy: epoch
num_train_epochs: 10
seed: 0
warmup_epochs: 1.0
remove_unused_columns: false
push_to_hub: true
hub_model_id: Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
hub_strategy: end
hub_private_repo: true
method_args:
gamma: 1.0
alpha: 1.0
retain_loss_type: NLL
beta: 0.1
data:
forget:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${forget_split}
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
retain:
TOFU_QA_retain:
handler: QADataset
args:
hf_args:
name: ${retain_split}
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
anchor: forget
collator:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
eval:
tofu:
metrics:
forget_truth_ratio:
pre_compute:
forget_Q_A_PARA_Prob:
datasets:
TOFU_QA_forget_para:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
forget_Q_A_PERT_Prob:
datasets:
TOFU_QA_forget_pert:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: closer_to_1_better
forget_quality:
pre_compute:
forget_truth_ratio:
pre_compute:
forget_Q_A_PARA_Prob:
datasets:
TOFU_QA_forget_para:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
forget_Q_A_PERT_Prob:
datasets:
TOFU_QA_forget_pert:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: closer_to_1_better
access_key: forget
reference_logs:
retain_model_logs:
path: ${eval.tofu.retain_logs_path}
include:
forget_truth_ratio:
access_key: retain
handler: ks_test
forget_Q_A_Prob:
datasets:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
forget_Q_A_ROUGE:
datasets:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
model_utility:
pre_compute:
retain_Q_A_Prob:
datasets:
TOFU_QA_retain_eval:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
retain_Q_A_ROUGE:
datasets:
TOFU_QA_retain_eval:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
retain_Truth_Ratio:
pre_compute:
retain_Q_A_PARA_Prob:
datasets:
TOFU_QA_retain_para:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
retain_Q_A_PERT_Prob:
datasets:
TOFU_QA_retain_pert:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: true_better
ra_Q_A_Prob_normalised:
pre_compute:
ra_Q_A_Prob:
datasets:
TOFU_QA_ra:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
ra_Q_A_PERT_Prob:
datasets:
TOFU_QA_ra_pert:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: probability_w_options
ra_Q_A_ROUGE:
datasets:
TOFU_QA_ra:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
ra_Truth_Ratio:
pre_compute:
ra_Q_A_Prob:
datasets:
TOFU_QA_ra:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
ra_Q_A_PERT_Prob:
datasets:
TOFU_QA_ra_pert:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: true_better
wf_Q_A_Prob_normalised:
pre_compute:
wf_Q_A_Prob:
datasets:
TOFU_QA_wf:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
wf_Q_A_PERT_Prob:
datasets:
TOFU_QA_wf_pert:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: probability_w_options
wf_Q_A_ROUGE:
datasets:
TOFU_QA_wf:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
wf_Truth_Ratio:
pre_compute:
wf_Q_A_Prob:
datasets:
TOFU_QA_wf:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
wf_Q_A_PERT_Prob:
datasets:
TOFU_QA_wf_pert:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: true_better
handler: hm_aggregate
privleak:
pre_compute:
mia_min_k:
datasets:
TOFU_QA_forget:
access_key: forget
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
TOFU_QA_holdout:
access_key: holdout
handler: QADataset
args:
hf_args:
name: ${eval.tofu.holdout_split}
path: locuslab/TOFU
split: train
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
batch_size: ${eval.tofu.batch_size}
handler: mia_min_k
k: 0.4
access_key: forget
reference_logs:
retain_model_logs:
path: ${eval.tofu.retain_logs_path}
include:
mia_min_k:
access_key: retain
handler: privleak
ref_value: 0.5
extraction_strength:
datasets:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: extraction_strength
batch_size: ${eval.tofu.batch_size}
handler: TOFUEvaluator
output_dir: ${paths.output_dir}
overwrite: true
forget_split: ${forget_split}
holdout_split: ${holdout_split}
retain_logs_path: ${retain_logs_path}
question_key: ${question_key}
batch_size: 32
paths:
root_dir: .
data_dir: ${paths.root_dir}/data/
datasets: ${paths.root_dir}/configs/data/datasets
output_dir: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
work_dir: ${hydra:runtime.cwd}
forget_split: forget10
retain_split: retain90
holdout_split: holdout10
retain_logs_path: null
question_key: question
task_name: tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
mode: unlearn

292
.hydra/hydra.yaml Normal file
View File

@@ -0,0 +1,292 @@
hydra:
run:
dir: ${paths.output_dir}
sweep:
dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
subdir: ${hydra.job.num}
launcher:
_target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
sweeper:
_target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
max_batch_size: null
params: null
help:
app_name: ${hydra.job.name}
header: '${hydra.help.app_name} is powered by Hydra.
'
footer: 'Powered by Hydra (https://hydra.cc)
Use --hydra-help to view Hydra specific help
'
template: '${hydra.help.header}
== Configuration groups ==
Compose your configuration from those groups (group=option)
$APP_CONFIG_GROUPS
== Config ==
Override anything in the config (foo.bar=value)
$CONFIG
${hydra.help.footer}
'
hydra_help:
template: 'Hydra (${hydra.runtime.version})
See https://hydra.cc for more info.
== Flags ==
$FLAGS_HELP
== Configuration groups ==
Compose your configuration from those groups (For example, append hydra/job_logging=disabled
to command line)
$HYDRA_CONFIG_GROUPS
Use ''--cfg hydra'' to Show the Hydra config.
'
hydra_help: ???
hydra_logging:
version: 1
formatters:
colorlog:
(): colorlog.ColoredFormatter
format: '[%(cyan)s%(asctime)s%(reset)s][%(purple)sHYDRA%(reset)s] %(message)s'
handlers:
console:
class: logging.StreamHandler
formatter: colorlog
stream: ext://sys.stdout
root:
level: INFO
handlers:
- console
disable_existing_loggers: false
job_logging:
version: 1
formatters:
simple:
format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
colorlog:
(): colorlog.ColoredFormatter
format: '[%(cyan)s%(asctime)s%(reset)s][%(blue)s%(name)s%(reset)s][%(log_color)s%(levelname)s%(reset)s]
- %(message)s'
log_colors:
DEBUG: purple
INFO: green
WARNING: yellow
ERROR: red
CRITICAL: red
handlers:
console:
class: logging.StreamHandler
formatter: colorlog
stream: ext://sys.stdout
file:
class: logging.FileHandler
formatter: simple
filename: ${hydra.runtime.output_dir}/${trainer.handler}.log
root:
level: INFO
handlers:
- console
- file
disable_existing_loggers: false
env: {}
mode: RUN
searchpath: []
callbacks: {}
output_subdir: .hydra
overrides:
hydra:
- hydra.mode=RUN
task:
- experiment=unlearn/tofu/default.yaml
- trainer=NPO
- task_name=tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
- model=Llama-3.2-1B-Instruct
- forget_split=forget10
- retain_split=retain90
- holdout_split=holdout10
- model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full
- model.model_args.attn_implementation=sdpa
- trainer.args.per_device_train_batch_size=4
- trainer.args.gradient_accumulation_steps=4
- trainer.args.ddp_find_unused_parameters=true
- trainer.args.do_eval=false
- trainer.args.save_strategy=no
- +trainer.args.push_to_hub=true
- +trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
- +trainer.args.hub_strategy=end
- +trainer.args.hub_private_repo=true
- paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
job:
name: train
chdir: null
override_dirname: +trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off,+trainer.args.hub_private_repo=true,+trainer.args.hub_strategy=end,+trainer.args.push_to_hub=true,experiment=unlearn/tofu/default.yaml,forget_split=forget10,holdout_split=holdout10,model.model_args.attn_implementation=sdpa,model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full,model=Llama-3.2-1B-Instruct,paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off,retain_split=retain90,task_name=tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off,trainer.args.ddp_find_unused_parameters=true,trainer.args.do_eval=false,trainer.args.gradient_accumulation_steps=4,trainer.args.per_device_train_batch_size=4,trainer.args.save_strategy=no,trainer=NPO
id: ???
num: ???
config_name: unlearn.yaml
env_set: {}
env_copy: []
config:
override_dirname:
kv_sep: '='
item_sep: ','
exclude_keys: []
runtime:
version: 1.3.0
version_base: '1.3'
cwd: /home/yonsei_jong/open-unlearning
config_sources:
- path: hydra.conf
schema: pkg
provider: hydra
- path: /home/yonsei_jong/open-unlearning/configs
schema: file
provider: main
- path: hydra_plugins.hydra_colorlog.conf
schema: pkg
provider: hydra-colorlog
- path: ''
schema: structured
provider: schema
output_dir: /home/yonsei_jong/open-unlearning/saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
choices:
experiment: unlearn/tofu/default.yaml
paths: default
hydra: default
eval: tofu
eval/tofu_metrics/../../collator@eval.tofu.metrics.extraction_strength.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.extraction_strength.datasets: TOFU_QA_forget
eval/tofu_metrics/.@eval.tofu.metrics.privleak.pre_compute.mia_min_k: mia_min_k
eval/tofu_metrics/./../../collator@eval.tofu.metrics.privleak.pre_compute.mia_min_k.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.privleak.pre_compute.mia_min_k.datasets: TOFU_MIA
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio: wf_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.datasets
: TOFU_QA_wf_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.datasets
: TOFU_QA_wf
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE: wf_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.datasets: TOFU_QA_wf
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised: wf_Q_A_Prob_normalised
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.datasets
: TOFU_QA_wf_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.datasets
: TOFU_QA_wf
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio: ra_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.datasets
: TOFU_QA_ra_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.datasets
: TOFU_QA_ra
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE: ra_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.datasets: TOFU_QA_ra
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised: ra_Q_A_Prob_normalised
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.datasets
: TOFU_QA_ra_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.datasets
: TOFU_QA_ra
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio: retain_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob: retain_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.datasets
: TOFU_QA_retain_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob: retain_Q_A_PARA_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.datasets
: TOFU_QA_retain_para
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE: retain_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.datasets: TOFU_QA_retain_eval
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob: retain_Q_A_Prob
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.datasets: TOFU_QA_retain_eval
eval/tofu_metrics/../../generation@eval.tofu.metrics.forget_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_ROUGE.datasets: TOFU_QA_forget
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_Prob.datasets: TOFU_QA_forget
eval/tofu_metrics/.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio: forget_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets
: TOFU_QA_forget_pert
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets
: TOFU_QA_forget_para
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets: TOFU_QA_forget_pert
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets: TOFU_QA_forget_para
collator: DataCollatorForSupervisedDataset
data: unlearn
data/datasets@data.eval: null
data/datasets@data.retain: TOFU_QA_retain
data/datasets@data.forget: TOFU_QA_forget
trainer: NPO
model: Llama-3.2-1B-Instruct
hydra/env: default
hydra/callbacks: null
hydra/job_logging: colorlog
hydra/hydra_logging: colorlog
hydra/hydra_help: default
hydra/help: default
hydra/sweeper: basic
hydra/launcher: basic
hydra/output: default
verbose: false

19
.hydra/overrides.yaml Normal file
View File

@@ -0,0 +1,19 @@
- experiment=unlearn/tofu/default.yaml
- trainer=NPO
- task_name=tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
- model=Llama-3.2-1B-Instruct
- forget_split=forget10
- retain_split=retain90
- holdout_split=holdout10
- model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full
- model.model_args.attn_implementation=sdpa
- trainer.args.per_device_train_batch_size=4
- trainer.args.gradient_accumulation_steps=4
- trainer.args.ddp_find_unused_parameters=true
- trainer.args.do_eval=false
- trainer.args.save_strategy=no
- +trainer.args.push_to_hub=true
- +trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
- +trainer.args.hub_strategy=end
- +trainer.args.hub_private_repo=true
- paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off

466
NPO.log Normal file
View File

@@ -0,0 +1,466 @@
[2026-05-18 13:54:31,762][model][WARNING] - Model open-unlearning/tofu_Llama-3.2-1B-Instruct_full requested with {'attn_implementation': 'flash_attention_2'}
[2026-05-18 13:54:57,079][model][INFO] - Setting pad_token as eos token: <|eot_id|>
[2026-05-18 13:55:00,796][evaluator][INFO] - Evaluations stored in the experiment directory: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
[2026-05-18 13:55:01,818][trainer][INFO] - NPO Trainer loaded, output_dir: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
[2026-05-18 13:55:02,370][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-18 13:55:02,370][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-0/evals/TOFU_EVAL.json
[2026-05-18 13:55:02,370][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-0/evals/TOFU_SUMMARY.json
[2026-05-18 13:55:03,973][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-18 13:55:20,129][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-18 13:55:53,342][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-18 13:55:53,343][evaluator][INFO] - Result for metric forget_truth_ratio: 0.4751472517287268
[2026-05-18 13:55:53,349][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-18 13:55:53,350][metrics][INFO] - Evaluating forget_quality
[2026-05-18 13:55:53,350][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-18 13:55:53,350][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-18 13:55:55,130][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-18 13:55:59,596][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.8804745058715343
[2026-05-18 13:56:00,882][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-18 13:56:47,342][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.8224148587081459
[2026-05-18 13:56:48,995][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-18 13:56:55,825][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-18 13:57:06,460][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-18 13:57:12,461][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-18 13:57:32,576][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-18 13:57:34,236][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-18 13:57:36,995][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-18 13:57:39,580][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-18 13:57:40,827][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-18 13:57:47,676][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-18 13:57:47,676][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-18 13:57:47,677][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-18 13:57:49,344][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-18 13:57:51,569][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-18 13:57:55,229][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-18 13:57:56,601][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-18 13:58:07,839][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-18 13:58:07,839][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-18 13:58:07,839][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-18 13:58:07,840][metrics][INFO] - Evaluating model_utility
[2026-05-18 13:58:07,840][evaluator][INFO] - Result for metric model_utility: 0.5986573345007462
[2026-05-18 13:58:10,704][metrics][INFO] - Evaluating mia_min_k
[2026-05-18 13:58:13,403][metrics][INFO] - Evaluating privleak
[2026-05-18 13:58:13,404][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-18 13:58:13,404][evaluator][INFO] - Result for metric privleak: -99.33374998013325
[2026-05-18 13:58:14,694][metrics][INFO] - Evaluating extraction_strength
[2026-05-18 13:58:15,308][evaluator][INFO] - Result for metric extraction_strength: 0.7035078413166893
[2026-05-18 14:00:28,716][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-18 14:00:28,716][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-25/evals/TOFU_EVAL.json
[2026-05-18 14:00:28,716][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-25/evals/TOFU_SUMMARY.json
[2026-05-18 14:00:30,797][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-18 14:00:34,916][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-18 14:00:48,840][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-18 14:00:48,841][evaluator][INFO] - Result for metric forget_truth_ratio: 0.5215462041054256
[2026-05-18 14:00:48,847][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-18 14:00:48,847][metrics][INFO] - Evaluating forget_quality
[2026-05-18 14:00:48,848][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-18 14:00:48,848][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-18 14:00:50,789][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-18 14:00:53,593][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.7682410891354085
[2026-05-18 14:00:54,905][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-18 14:01:00,411][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.6679393669272393
[2026-05-18 14:01:02,103][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-18 14:01:06,134][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-18 14:01:11,627][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-18 14:01:16,346][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-18 14:01:30,082][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-18 14:01:31,788][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-18 14:01:33,753][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-18 14:01:35,615][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-18 14:01:36,873][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-18 14:01:37,587][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-18 14:01:37,587][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:01:37,587][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-18 14:01:38,856][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-18 14:01:40,600][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-18 14:01:42,178][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-18 14:01:43,527][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-18 14:01:44,439][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-18 14:01:44,439][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:01:44,439][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-18 14:01:44,440][metrics][INFO] - Evaluating model_utility
[2026-05-18 14:01:44,440][evaluator][INFO] - Result for metric model_utility: 0.5802270741620733
[2026-05-18 14:01:47,079][metrics][INFO] - Evaluating mia_min_k
[2026-05-18 14:01:48,071][metrics][INFO] - Evaluating privleak
[2026-05-18 14:01:48,071][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-18 14:01:48,071][evaluator][INFO] - Result for metric privleak: -98.86124998022777
[2026-05-18 14:01:49,359][metrics][INFO] - Evaluating extraction_strength
[2026-05-18 14:01:50,046][evaluator][INFO] - Result for metric extraction_strength: 0.4727573242328045
[2026-05-18 14:02:43,850][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-18 14:02:43,850][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-50/evals/TOFU_EVAL.json
[2026-05-18 14:02:43,850][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-50/evals/TOFU_SUMMARY.json
[2026-05-18 14:02:45,429][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-18 14:02:49,505][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-18 14:03:03,449][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-18 14:03:03,450][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6758352799741583
[2026-05-18 14:03:03,456][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-18 14:03:03,456][metrics][INFO] - Evaluating forget_quality
[2026-05-18 14:03:03,456][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-18 14:03:03,456][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-18 14:03:05,160][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-18 14:03:07,961][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.23828835071995855
[2026-05-18 14:03:09,224][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-18 14:03:35,722][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.266978506986982
[2026-05-18 14:03:37,475][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-18 14:03:41,707][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-18 14:03:45,668][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-18 14:03:49,814][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-18 14:04:03,639][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-18 14:04:05,383][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-18 14:04:07,352][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-18 14:04:09,228][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-18 14:04:10,572][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-18 14:04:11,267][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-18 14:04:11,267][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:04:11,267][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-18 14:04:13,555][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-18 14:04:15,327][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-18 14:04:16,908][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-18 14:04:18,170][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-18 14:04:18,873][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-18 14:04:18,874][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:04:18,874][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-18 14:04:18,874][metrics][INFO] - Evaluating model_utility
[2026-05-18 14:04:18,875][evaluator][INFO] - Result for metric model_utility: 0.365025395428697
[2026-05-18 14:04:21,447][metrics][INFO] - Evaluating mia_min_k
[2026-05-18 14:04:22,456][metrics][INFO] - Evaluating privleak
[2026-05-18 14:04:22,457][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-18 14:04:22,457][evaluator][INFO] - Result for metric privleak: -75.04874998499025
[2026-05-18 14:04:23,844][metrics][INFO] - Evaluating extraction_strength
[2026-05-18 14:04:24,658][evaluator][INFO] - Result for metric extraction_strength: 0.08614304082608436
[2026-05-18 14:05:19,095][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-18 14:05:19,095][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-75/evals/TOFU_EVAL.json
[2026-05-18 14:05:19,095][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-75/evals/TOFU_SUMMARY.json
[2026-05-18 14:05:20,882][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-18 14:05:25,058][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-18 14:05:39,034][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-18 14:05:39,035][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6668836791633079
[2026-05-18 14:05:39,041][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-18 14:05:39,042][metrics][INFO] - Evaluating forget_quality
[2026-05-18 14:05:39,042][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-18 14:05:39,042][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-18 14:05:40,713][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-18 14:05:43,524][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.22939431543461977
[2026-05-18 14:05:44,787][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-18 14:05:48,594][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.19464342599083026
[2026-05-18 14:05:49,944][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-18 14:05:53,959][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-18 14:05:59,042][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-18 14:06:03,088][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-18 14:06:16,837][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-18 14:06:18,531][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-18 14:06:20,445][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-18 14:06:22,314][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-18 14:06:23,557][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-18 14:06:24,902][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-18 14:06:24,903][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:06:24,903][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-18 14:06:26,318][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-18 14:06:28,106][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-18 14:06:29,685][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-18 14:06:31,014][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-18 14:06:31,984][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-18 14:06:31,984][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:06:31,984][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-18 14:06:31,984][metrics][INFO] - Evaluating model_utility
[2026-05-18 14:06:31,985][evaluator][INFO] - Result for metric model_utility: 0.34932386649395714
[2026-05-18 14:06:34,498][metrics][INFO] - Evaluating mia_min_k
[2026-05-18 14:06:35,497][metrics][INFO] - Evaluating privleak
[2026-05-18 14:06:35,498][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-18 14:06:35,498][evaluator][INFO] - Result for metric privleak: -65.46124998690773
[2026-05-18 14:06:36,810][metrics][INFO] - Evaluating extraction_strength
[2026-05-18 14:06:37,620][evaluator][INFO] - Result for metric extraction_strength: 0.09391103318955707
[2026-05-18 14:07:30,784][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-18 14:07:30,785][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-100/evals/TOFU_EVAL.json
[2026-05-18 14:07:30,785][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-100/evals/TOFU_SUMMARY.json
[2026-05-18 14:07:32,467][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-18 14:07:36,654][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-18 14:07:50,605][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-18 14:07:50,606][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6531148071926944
[2026-05-18 14:07:50,612][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-18 14:07:50,612][metrics][INFO] - Evaluating forget_quality
[2026-05-18 14:07:50,613][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-18 14:07:50,613][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-18 14:07:52,345][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-18 14:07:55,159][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.1875428356032353
[2026-05-18 14:07:56,408][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-18 14:08:01,500][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.22033509820856984
[2026-05-18 14:08:03,199][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-18 14:08:07,214][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-18 14:08:13,022][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-18 14:08:17,053][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-18 14:08:30,813][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-18 14:08:32,534][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-18 14:08:34,570][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-18 14:08:36,441][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-18 14:08:37,679][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-18 14:08:39,458][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-18 14:08:39,458][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:08:39,458][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-18 14:08:40,743][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-18 14:08:42,502][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-18 14:08:44,083][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-18 14:08:45,415][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-18 14:08:46,470][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-18 14:08:46,470][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:08:46,470][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-18 14:08:46,471][metrics][INFO] - Evaluating model_utility
[2026-05-18 14:08:46,471][evaluator][INFO] - Result for metric model_utility: 0.43717298699454965
[2026-05-18 14:08:48,984][metrics][INFO] - Evaluating mia_min_k
[2026-05-18 14:08:49,984][metrics][INFO] - Evaluating privleak
[2026-05-18 14:08:49,984][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-18 14:08:49,985][evaluator][INFO] - Result for metric privleak: -38.19124999236173
[2026-05-18 14:08:51,268][metrics][INFO] - Evaluating extraction_strength
[2026-05-18 14:08:52,071][evaluator][INFO] - Result for metric extraction_strength: 0.09873176679531875
[2026-05-18 14:09:41,834][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-18 14:09:41,834][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-125/evals/TOFU_EVAL.json
[2026-05-18 14:09:41,834][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-125/evals/TOFU_SUMMARY.json
[2026-05-18 14:09:43,508][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-18 14:09:47,641][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-18 14:10:01,592][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-18 14:10:01,592][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6237154687561808
[2026-05-18 14:10:01,598][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-18 14:10:01,599][metrics][INFO] - Evaluating forget_quality
[2026-05-18 14:10:01,599][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-18 14:10:01,599][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-18 14:10:03,260][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-18 14:10:06,063][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.15545873703609686
[2026-05-18 14:10:07,313][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-18 14:10:12,073][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.2513310748335476
[2026-05-18 14:10:13,325][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-18 14:10:17,327][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-18 14:10:23,249][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-18 14:10:27,286][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-18 14:10:41,035][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-18 14:10:42,745][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-18 14:10:44,708][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-18 14:10:46,577][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-18 14:10:47,822][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-18 14:10:48,838][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-18 14:10:48,838][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:10:48,838][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-18 14:10:50,197][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-18 14:10:52,000][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-18 14:10:53,577][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-18 14:10:54,817][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-18 14:10:56,881][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-18 14:10:56,882][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:10:56,882][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-18 14:10:56,882][metrics][INFO] - Evaluating model_utility
[2026-05-18 14:10:56,883][evaluator][INFO] - Result for metric model_utility: 0.49362893415678766
[2026-05-18 14:10:59,475][metrics][INFO] - Evaluating mia_min_k
[2026-05-18 14:11:00,462][metrics][INFO] - Evaluating privleak
[2026-05-18 14:11:00,462][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-18 14:11:00,462][evaluator][INFO] - Result for metric privleak: -13.094999997381013
[2026-05-18 14:11:01,746][metrics][INFO] - Evaluating extraction_strength
[2026-05-18 14:11:02,542][evaluator][INFO] - Result for metric extraction_strength: 0.09858533284264409
[2026-05-18 14:11:53,342][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-18 14:11:53,343][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-150/evals/TOFU_EVAL.json
[2026-05-18 14:11:53,343][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-150/evals/TOFU_SUMMARY.json
[2026-05-18 14:11:55,079][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-18 14:11:59,252][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-18 14:12:13,207][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-18 14:12:13,207][evaluator][INFO] - Result for metric forget_truth_ratio: 0.5806421898784745
[2026-05-18 14:12:13,213][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-18 14:12:13,214][metrics][INFO] - Evaluating forget_quality
[2026-05-18 14:12:13,214][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-18 14:12:13,214][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-18 14:12:14,796][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-18 14:12:17,599][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.15392853119468783
[2026-05-18 14:12:18,838][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-18 14:12:23,915][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.27986895187799854
[2026-05-18 14:12:25,549][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-18 14:12:29,551][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-18 14:12:35,561][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-18 14:12:39,971][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-18 14:12:53,737][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-18 14:12:55,355][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-18 14:12:57,221][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-18 14:12:59,097][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-18 14:13:00,360][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-18 14:13:01,499][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-18 14:13:01,499][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:13:01,499][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-18 14:13:02,741][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-18 14:13:04,731][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-18 14:13:06,308][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-18 14:13:07,535][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-18 14:13:08,742][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-18 14:13:08,742][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:13:08,742][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-18 14:13:08,743][metrics][INFO] - Evaluating model_utility
[2026-05-18 14:13:08,743][evaluator][INFO] - Result for metric model_utility: 0.5353537708875081
[2026-05-18 14:13:11,314][metrics][INFO] - Evaluating mia_min_k
[2026-05-18 14:13:12,318][metrics][INFO] - Evaluating privleak
[2026-05-18 14:13:12,318][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-18 14:13:12,318][evaluator][INFO] - Result for metric privleak: -21.444999995711004
[2026-05-18 14:13:13,720][metrics][INFO] - Evaluating extraction_strength
[2026-05-18 14:13:14,523][evaluator][INFO] - Result for metric extraction_strength: 0.10802013307923694
[2026-05-18 14:14:07,060][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-18 14:14:07,060][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-175/evals/TOFU_EVAL.json
[2026-05-18 14:14:07,060][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-175/evals/TOFU_SUMMARY.json
[2026-05-18 14:14:08,719][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-18 14:14:12,789][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-18 14:14:26,764][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-18 14:14:26,764][evaluator][INFO] - Result for metric forget_truth_ratio: 0.5689891586427206
[2026-05-18 14:14:26,770][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-18 14:14:26,771][metrics][INFO] - Evaluating forget_quality
[2026-05-18 14:14:26,771][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-18 14:14:26,771][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-18 14:14:28,494][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-18 14:14:31,298][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.14874292317486834
[2026-05-18 14:14:32,623][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-18 14:14:38,203][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.2956713274905
[2026-05-18 14:14:39,847][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-18 14:14:43,836][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-18 14:14:50,807][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-18 14:14:54,853][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-18 14:15:08,629][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-18 14:15:10,462][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-18 14:15:12,391][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-18 14:15:14,266][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-18 14:15:15,515][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-18 14:15:16,487][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-18 14:15:16,487][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:15:16,488][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-18 14:15:17,760][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-18 14:15:19,489][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-18 14:15:21,071][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-18 14:15:22,347][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-18 14:15:24,289][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-18 14:15:24,289][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:15:24,289][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-18 14:15:24,290][metrics][INFO] - Evaluating model_utility
[2026-05-18 14:15:24,290][evaluator][INFO] - Result for metric model_utility: 0.5515531045256538
[2026-05-18 14:15:26,783][metrics][INFO] - Evaluating mia_min_k
[2026-05-18 14:15:27,785][metrics][INFO] - Evaluating privleak
[2026-05-18 14:15:27,785][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-18 14:15:27,785][evaluator][INFO] - Result for metric privleak: -15.800624996839888
[2026-05-18 14:15:29,043][metrics][INFO] - Evaluating extraction_strength
[2026-05-18 14:15:29,839][evaluator][INFO] - Result for metric extraction_strength: 0.11600868539246466
[2026-05-18 14:16:24,603][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-18 14:16:24,603][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-200/evals/TOFU_EVAL.json
[2026-05-18 14:16:24,603][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-200/evals/TOFU_SUMMARY.json
[2026-05-18 14:16:26,252][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-18 14:16:30,454][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-18 14:16:44,412][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-18 14:16:44,413][evaluator][INFO] - Result for metric forget_truth_ratio: 0.5614269492250329
[2026-05-18 14:16:44,419][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-18 14:16:44,419][metrics][INFO] - Evaluating forget_quality
[2026-05-18 14:16:44,419][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-18 14:16:44,419][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-18 14:16:46,111][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-18 14:16:48,925][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.14210461506576394
[2026-05-18 14:16:50,235][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-18 14:16:55,885][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.2970728548646609
[2026-05-18 14:16:57,523][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-18 14:17:01,535][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-18 14:17:08,651][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-18 14:17:12,659][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-18 14:17:26,399][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-18 14:17:28,037][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-18 14:17:29,907][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-18 14:17:31,779][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-18 14:17:33,006][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-18 14:17:33,823][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-18 14:17:33,823][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:17:33,823][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-18 14:17:35,264][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-18 14:17:36,972][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-18 14:17:38,549][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-18 14:17:39,786][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-18 14:17:43,814][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-18 14:17:43,815][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:17:43,815][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-18 14:17:43,815][metrics][INFO] - Evaluating model_utility
[2026-05-18 14:17:43,816][evaluator][INFO] - Result for metric model_utility: 0.5591748848281174
[2026-05-18 14:17:46,742][metrics][INFO] - Evaluating mia_min_k
[2026-05-18 14:17:47,745][metrics][INFO] - Evaluating privleak
[2026-05-18 14:17:47,745][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-18 14:17:47,745][evaluator][INFO] - Result for metric privleak: -17.166249996566755
[2026-05-18 14:17:49,081][metrics][INFO] - Evaluating extraction_strength
[2026-05-18 14:17:49,881][evaluator][INFO] - Result for metric extraction_strength: 0.11815252416178619
[2026-05-18 14:18:39,514][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-18 14:18:39,514][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-225/evals/TOFU_EVAL.json
[2026-05-18 14:18:39,514][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-225/evals/TOFU_SUMMARY.json
[2026-05-18 14:18:41,139][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-18 14:18:45,205][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-18 14:18:59,191][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-18 14:18:59,193][evaluator][INFO] - Result for metric forget_truth_ratio: 0.559432289746286
[2026-05-18 14:18:59,199][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-18 14:18:59,199][metrics][INFO] - Evaluating forget_quality
[2026-05-18 14:18:59,199][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-18 14:18:59,199][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-18 14:19:00,827][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-18 14:19:03,629][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.13278577822376975
[2026-05-18 14:19:04,861][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-18 14:19:10,156][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.30219775268292687
[2026-05-18 14:19:11,820][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-18 14:19:15,860][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-18 14:19:22,552][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-18 14:19:26,622][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-18 14:19:40,363][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-18 14:19:42,025][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-18 14:19:44,233][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-18 14:19:46,108][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-18 14:19:47,431][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-18 14:19:48,347][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-18 14:19:48,348][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:19:48,348][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-18 14:19:50,409][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-18 14:19:52,353][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-18 14:19:53,929][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-18 14:19:55,162][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-18 14:19:56,423][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-18 14:19:56,423][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:19:56,423][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-18 14:19:56,423][metrics][INFO] - Evaluating model_utility
[2026-05-18 14:19:56,424][evaluator][INFO] - Result for metric model_utility: 0.5593146400206448
[2026-05-18 14:19:59,178][metrics][INFO] - Evaluating mia_min_k
[2026-05-18 14:20:00,182][metrics][INFO] - Evaluating privleak
[2026-05-18 14:20:00,183][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-18 14:20:00,183][evaluator][INFO] - Result for metric privleak: -14.034999997192998
[2026-05-18 14:20:01,562][metrics][INFO] - Evaluating extraction_strength
[2026-05-18 14:20:02,368][evaluator][INFO] - Result for metric extraction_strength: 0.11882007020428687
[2026-05-18 14:20:54,400][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-18 14:20:54,400][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-250/evals/TOFU_EVAL.json
[2026-05-18 14:20:54,400][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off/checkpoint-250/evals/TOFU_SUMMARY.json
[2026-05-18 14:20:56,086][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-18 14:21:00,205][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-18 14:21:14,170][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-18 14:21:14,170][evaluator][INFO] - Result for metric forget_truth_ratio: 0.5612359669808998
[2026-05-18 14:21:14,176][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-18 14:21:14,177][metrics][INFO] - Evaluating forget_quality
[2026-05-18 14:21:14,177][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-18 14:21:14,177][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-18 14:21:15,906][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-18 14:21:18,726][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.13566936177085154
[2026-05-18 14:21:20,061][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-18 14:21:25,547][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3076188844200944
[2026-05-18 14:21:27,366][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-18 14:21:31,504][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-18 14:21:38,204][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-18 14:21:42,261][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-18 14:21:56,026][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-18 14:21:57,712][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-18 14:21:59,628][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-18 14:22:01,503][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-18 14:22:02,770][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-18 14:22:03,597][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-18 14:22:03,597][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:22:03,597][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-18 14:22:04,849][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-18 14:22:06,654][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-18 14:22:08,232][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-18 14:22:09,479][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-18 14:22:10,670][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-18 14:22:10,671][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-18 14:22:10,671][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-18 14:22:10,671][metrics][INFO] - Evaluating model_utility
[2026-05-18 14:22:10,671][evaluator][INFO] - Result for metric model_utility: 0.5599261205924377
[2026-05-18 14:22:13,279][metrics][INFO] - Evaluating mia_min_k
[2026-05-18 14:22:14,284][metrics][INFO] - Evaluating privleak
[2026-05-18 14:22:14,284][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-18 14:22:14,284][evaluator][INFO] - Result for metric privleak: -13.192499997361507
[2026-05-18 14:22:15,573][metrics][INFO] - Evaluating extraction_strength
[2026-05-18 14:22:16,382][evaluator][INFO] - Result for metric extraction_strength: 0.11907237943929123

56
README.md Normal file
View File

@@ -0,0 +1,56 @@
---
library_name: transformers
license: bsd-3-clause
base_model: open-unlearning/tofu_Llama-3.2-1B-Instruct_full
tags:
- generated_from_trainer
model-index:
- name: tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
results: []
---
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
should probably proofread and complete it, then remove this comment. -->
# tofu_Llama-3.2-1B-Instruct_forget10_NPO_qat-off
This model is a fine-tuned version of [open-unlearning/tofu_Llama-3.2-1B-Instruct_full](https://huggingface.co/open-unlearning/tofu_Llama-3.2-1B-Instruct_full) on an unknown dataset.
## Model description
More information needed
## Intended uses & limitations
More information needed
## Training and evaluation data
More information needed
## Training procedure
### Training hyperparameters
The following hyperparameters were used during training:
- learning_rate: 1e-05
- train_batch_size: 4
- eval_batch_size: 16
- seed: 0
- gradient_accumulation_steps: 4
- total_train_batch_size: 16
- optimizer: Use OptimizerNames.PAGED_ADAMW with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
- lr_scheduler_type: linear
- lr_scheduler_warmup_steps: 25
- num_epochs: 10
### Training results
### Framework versions
- Transformers 4.51.3
- Pytorch 2.11.0+cu128
- Datasets 3.0.1
- Tokenizers 0.21.4

39
config.json Normal file
View File

@@ -0,0 +1,39 @@
{
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 128000,
"eos_token_id": [
128001,
128008,
128009
],
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 2048,
"initializer_range": 0.02,
"intermediate_size": 8192,
"max_position_embeddings": 131072,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 16,
"num_key_value_heads": 8,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": {
"factor": 32.0,
"high_freq_factor": 4.0,
"low_freq_factor": 1.0,
"original_max_position_embeddings": 8192,
"rope_type": "llama3"
},
"rope_theta": 500000.0,
"tie_word_embeddings": true,
"torch_dtype": "bfloat16",
"transformers_version": "4.51.3",
"use_cache": true,
"vocab_size": 128256
}

12
generation_config.json Normal file
View File

@@ -0,0 +1,12 @@
{
"bos_token_id": 128000,
"do_sample": true,
"eos_token_id": [
128001,
128008,
128009
],
"temperature": 0.6,
"top_p": 0.9,
"transformers_version": "4.51.3"
}

3
logs/events.out.tfevents.1779080102.jeesup.1839598.0 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:78551bf7b0d2fde3d73638759f16ad061b1495256741ea31e63b8fc1abb48019
size 20636

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7383e06057d2d83afc34628a618c8a6ef315e21e9369dd551f8d842636ef0947
size 2471645608

17
special_tokens_map.json Normal file
View File

@@ -0,0 +1,17 @@
{
"bos_token": {
"content": "<|begin_of_text|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|eot_id|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": "<|eot_id|>"
}

BIN
tokenizer.json (Stored with Git LFS) Normal file
View File

Binary file not shown.

2064
tokenizer_config.json Normal file
View File

File diff suppressed because it is too large Load Diff

503
trainer_state.json Normal file
View File

@@ -0,0 +1,503 @@
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 10.0,
"eval_steps": 500,
"global_step": 250,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0,
"extraction_strength": 0.7035078413166893,
"forget_Q_A_Prob": 0.8804745058715343,
"forget_Q_A_ROUGE": 0.8224148587081459,
"forget_truth_ratio": 0.4751472517287268,
"model_utility": 0.5986573345007462,
"privleak": -99.33374998013325,
"step": 0
},
{
"epoch": 0.2,
"grad_norm": 1176.0,
"learning_rate": 1.6000000000000001e-06,
"loss": 55.8963,
"step": 5
},
{
"epoch": 0.4,
"grad_norm": 1064.0,
"learning_rate": 3.6000000000000003e-06,
"loss": 55.7682,
"step": 10
},
{
"epoch": 0.6,
"grad_norm": 1184.0,
"learning_rate": 5.600000000000001e-06,
"loss": 55.3329,
"step": 15
},
{
"epoch": 0.8,
"grad_norm": 1168.0,
"learning_rate": 7.600000000000001e-06,
"loss": 53.5387,
"step": 20
},
{
"epoch": 1.0,
"grad_norm": 1144.0,
"learning_rate": 9.600000000000001e-06,
"loss": 47.16,
"step": 25
},
{
"epoch": 1.0,
"extraction_strength": 0.4727573242328045,
"forget_Q_A_Prob": 0.7682410891354085,
"forget_Q_A_ROUGE": 0.6679393669272393,
"forget_truth_ratio": 0.5215462041054256,
"model_utility": 0.5802270741620733,
"privleak": -98.86124998022777,
"step": 25
},
{
"epoch": 1.2,
"grad_norm": 732.0,
"learning_rate": 9.822222222222223e-06,
"loss": 32.0906,
"step": 30
},
{
"epoch": 1.4,
"grad_norm": 592.0,
"learning_rate": 9.600000000000001e-06,
"loss": 24.9835,
"step": 35
},
{
"epoch": 1.6,
"grad_norm": 400.0,
"learning_rate": 9.377777777777779e-06,
"loss": 15.8103,
"step": 40
},
{
"epoch": 1.8,
"grad_norm": 166.0,
"learning_rate": 9.155555555555557e-06,
"loss": 11.057,
"step": 45
},
{
"epoch": 2.0,
"grad_norm": 108.0,
"learning_rate": 8.933333333333333e-06,
"loss": 9.1845,
"step": 50
},
{
"epoch": 2.0,
"extraction_strength": 0.08614304082608436,
"forget_Q_A_Prob": 0.23828835071995855,
"forget_Q_A_ROUGE": 0.266978506986982,
"forget_truth_ratio": 0.6758352799741583,
"model_utility": 0.365025395428697,
"privleak": -75.04874998499025,
"step": 50
},
{
"epoch": 2.2,
"grad_norm": 120.5,
"learning_rate": 8.711111111111111e-06,
"loss": 9.094,
"step": 55
},
{
"epoch": 2.4,
"grad_norm": 202.0,
"learning_rate": 8.48888888888889e-06,
"loss": 7.5936,
"step": 60
},
{
"epoch": 2.6,
"grad_norm": 82.0,
"learning_rate": 8.266666666666667e-06,
"loss": 6.9892,
"step": 65
},
{
"epoch": 2.8,
"grad_norm": 84.0,
"learning_rate": 8.044444444444444e-06,
"loss": 6.5445,
"step": 70
},
{
"epoch": 3.0,
"grad_norm": 89.5,
"learning_rate": 7.822222222222224e-06,
"loss": 6.2284,
"step": 75
},
{
"epoch": 3.0,
"extraction_strength": 0.09391103318955707,
"forget_Q_A_Prob": 0.22939431543461977,
"forget_Q_A_ROUGE": 0.19464342599083026,
"forget_truth_ratio": 0.6668836791633079,
"model_utility": 0.34932386649395714,
"privleak": -65.46124998690773,
"step": 75
},
{
"epoch": 3.2,
"grad_norm": 129.0,
"learning_rate": 7.600000000000001e-06,
"loss": 5.9146,
"step": 80
},
{
"epoch": 3.4,
"grad_norm": 76.5,
"learning_rate": 7.377777777777778e-06,
"loss": 5.5015,
"step": 85
},
{
"epoch": 3.6,
"grad_norm": 104.0,
"learning_rate": 7.155555555555556e-06,
"loss": 5.37,
"step": 90
},
{
"epoch": 3.8,
"grad_norm": 91.5,
"learning_rate": 6.9333333333333344e-06,
"loss": 4.9139,
"step": 95
},
{
"epoch": 4.0,
"grad_norm": 118.5,
"learning_rate": 6.711111111111111e-06,
"loss": 4.9409,
"step": 100
},
{
"epoch": 4.0,
"extraction_strength": 0.09873176679531875,
"forget_Q_A_Prob": 0.1875428356032353,
"forget_Q_A_ROUGE": 0.22033509820856984,
"forget_truth_ratio": 0.6531148071926944,
"model_utility": 0.43717298699454965,
"privleak": -38.19124999236173,
"step": 100
},
{
"epoch": 4.2,
"grad_norm": 175.0,
"learning_rate": 6.488888888888889e-06,
"loss": 4.2452,
"step": 105
},
{
"epoch": 4.4,
"grad_norm": 144.0,
"learning_rate": 6.266666666666668e-06,
"loss": 4.0208,
"step": 110
},
{
"epoch": 4.6,
"grad_norm": 173.0,
"learning_rate": 6.044444444444445e-06,
"loss": 3.709,
"step": 115
},
{
"epoch": 4.8,
"grad_norm": 228.0,
"learning_rate": 5.822222222222223e-06,
"loss": 3.4964,
"step": 120
},
{
"epoch": 5.0,
"grad_norm": 220.0,
"learning_rate": 5.600000000000001e-06,
"loss": 3.3359,
"step": 125
},
{
"epoch": 5.0,
"extraction_strength": 0.09858533284264409,
"forget_Q_A_Prob": 0.15545873703609686,
"forget_Q_A_ROUGE": 0.2513310748335476,
"forget_truth_ratio": 0.6237154687561808,
"model_utility": 0.49362893415678766,
"privleak": -13.094999997381013,
"step": 125
},
{
"epoch": 5.2,
"grad_norm": 140.0,
"learning_rate": 5.3777777777777784e-06,
"loss": 2.7955,
"step": 130
},
{
"epoch": 5.4,
"grad_norm": 146.0,
"learning_rate": 5.155555555555556e-06,
"loss": 2.5476,
"step": 135
},
{
"epoch": 5.6,
"grad_norm": 147.0,
"learning_rate": 4.933333333333334e-06,
"loss": 2.3843,
"step": 140
},
{
"epoch": 5.8,
"grad_norm": 233.0,
"learning_rate": 4.711111111111111e-06,
"loss": 2.4493,
"step": 145
},
{
"epoch": 6.0,
"grad_norm": 173.0,
"learning_rate": 4.488888888888889e-06,
"loss": 2.2127,
"step": 150
},
{
"epoch": 6.0,
"extraction_strength": 0.10802013307923694,
"forget_Q_A_Prob": 0.15392853119468783,
"forget_Q_A_ROUGE": 0.27986895187799854,
"forget_truth_ratio": 0.5806421898784745,
"model_utility": 0.5353537708875081,
"privleak": -21.444999995711004,
"step": 150
},
{
"epoch": 6.2,
"grad_norm": 215.0,
"learning_rate": 4.266666666666668e-06,
"loss": 2.1857,
"step": 155
},
{
"epoch": 6.4,
"grad_norm": 105.5,
"learning_rate": 4.044444444444445e-06,
"loss": 1.7586,
"step": 160
},
{
"epoch": 6.6,
"grad_norm": 116.0,
"learning_rate": 3.8222222222222224e-06,
"loss": 1.9203,
"step": 165
},
{
"epoch": 6.8,
"grad_norm": 116.0,
"learning_rate": 3.6000000000000003e-06,
"loss": 1.8145,
"step": 170
},
{
"epoch": 7.0,
"grad_norm": 153.0,
"learning_rate": 3.377777777777778e-06,
"loss": 1.8083,
"step": 175
},
{
"epoch": 7.0,
"extraction_strength": 0.11600868539246466,
"forget_Q_A_Prob": 0.14874292317486834,
"forget_Q_A_ROUGE": 0.2956713274905,
"forget_truth_ratio": 0.5689891586427206,
"model_utility": 0.5515531045256538,
"privleak": -15.800624996839888,
"step": 175
},
{
"epoch": 7.2,
"grad_norm": 126.5,
"learning_rate": 3.1555555555555555e-06,
"loss": 1.5654,
"step": 180
},
{
"epoch": 7.4,
"grad_norm": 132.0,
"learning_rate": 2.9333333333333338e-06,
"loss": 1.6857,
"step": 185
},
{
"epoch": 7.6,
"grad_norm": 185.0,
"learning_rate": 2.7111111111111116e-06,
"loss": 1.583,
"step": 190
},
{
"epoch": 7.8,
"grad_norm": 214.0,
"learning_rate": 2.488888888888889e-06,
"loss": 1.6843,
"step": 195
},
{
"epoch": 8.0,
"grad_norm": 192.0,
"learning_rate": 2.266666666666667e-06,
"loss": 1.5761,
"step": 200
},
{
"epoch": 8.0,
"extraction_strength": 0.11815252416178619,
"forget_Q_A_Prob": 0.14210461506576394,
"forget_Q_A_ROUGE": 0.2970728548646609,
"forget_truth_ratio": 0.5614269492250329,
"model_utility": 0.5591748848281174,
"privleak": -17.166249996566755,
"step": 200
},
{
"epoch": 8.2,
"grad_norm": 205.0,
"learning_rate": 2.0444444444444447e-06,
"loss": 1.3298,
"step": 205
},
{
"epoch": 8.4,
"grad_norm": 114.0,
"learning_rate": 1.8222222222222225e-06,
"loss": 1.4877,
"step": 210
},
{
"epoch": 8.6,
"grad_norm": 169.0,
"learning_rate": 1.6000000000000001e-06,
"loss": 1.5678,
"step": 215
},
{
"epoch": 8.8,
"grad_norm": 144.0,
"learning_rate": 1.377777777777778e-06,
"loss": 1.4845,
"step": 220
},
{
"epoch": 9.0,
"grad_norm": 155.0,
"learning_rate": 1.1555555555555556e-06,
"loss": 1.594,
"step": 225
},
{
"epoch": 9.0,
"extraction_strength": 0.11882007020428687,
"forget_Q_A_Prob": 0.13278577822376975,
"forget_Q_A_ROUGE": 0.30219775268292687,
"forget_truth_ratio": 0.559432289746286,
"model_utility": 0.5593146400206448,
"privleak": -14.034999997192998,
"step": 225
},
{
"epoch": 9.2,
"grad_norm": 188.0,
"learning_rate": 9.333333333333334e-07,
"loss": 1.3878,
"step": 230
},
{
"epoch": 9.4,
"grad_norm": 120.5,
"learning_rate": 7.111111111111112e-07,
"loss": 1.4716,
"step": 235
},
{
"epoch": 9.6,
"grad_norm": 148.0,
"learning_rate": 4.88888888888889e-07,
"loss": 1.4323,
"step": 240
},
{
"epoch": 9.8,
"grad_norm": 163.0,
"learning_rate": 2.666666666666667e-07,
"loss": 1.5144,
"step": 245
},
{
"epoch": 10.0,
"grad_norm": 136.0,
"learning_rate": 4.444444444444445e-08,
"loss": 1.5361,
"step": 250
},
{
"epoch": 10.0,
"extraction_strength": 0.11907237943929123,
"forget_Q_A_Prob": 0.13566936177085154,
"forget_Q_A_ROUGE": 0.3076188844200944,
"forget_truth_ratio": 0.5612359669808998,
"model_utility": 0.5599261205924377,
"privleak": -13.192499997361507,
"step": 250
},
{
"epoch": 10.0,
"step": 250,
"total_flos": 0.0,
"train_loss": 9.749949935913087,
"train_runtime": 1634.0475,
"train_samples_per_second": 2.448,
"train_steps_per_second": 0.153
}
],
"logging_steps": 5,
"max_steps": 250,
"num_input_tokens_seen": 0,
"num_train_epochs": 10,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:6651b993466fcd718a617965411100d80f690eddf5395645b925259d7addc2eb
size 5841