Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-06-04 16:18:17 +08:00
commit 4aa37f6cc7
15 changed files with 4189 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.gitattributes vendored Normal file
View File

@@ -0,0 +1,36 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text

671
.hydra/config.yaml Normal file
View File

@@ -0,0 +1,671 @@
model:
model_args:
pretrained_model_name_or_path: open-unlearning/tofu_Llama-3.2-1B-Instruct_full
attn_implementation: sdpa
torch_dtype: bfloat16
tokenizer_args:
pretrained_model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
template_args:
apply_chat_template: true
system_prompt: You are a helpful assistant.
system_prompt_with_special_tokens: '<|begin_of_text|><|start_header_id|>system<|end_header_id|>
You are a helpful assistant.<|eot_id|>'
user_start_tag: '<|start_header_id|>user<|end_header_id|>
'
user_end_tag: <|eot_id|>
asst_start_tag: '<|start_header_id|>assistant<|end_header_id|>
'
asst_end_tag: <|eot_id|>
date_string: 10 Apr 2025
trainer:
handler: SimNPO
args:
per_device_train_batch_size: 4
per_device_eval_batch_size: 16
gradient_accumulation_steps: 4
learning_rate: 1.0e-05
bf16: true
bf16_full_eval: true
logging_steps: 5
output_dir: ${paths.output_dir}
logging_dir: ${trainer.args.output_dir}/logs
report_to: tensorboard
ddp_find_unused_parameters: true
gradient_checkpointing: false
optim: paged_adamw_32bit
save_strategy: 'no'
save_only_model: true
weight_decay: 0.01
do_train: true
do_eval: false
eval_on_start: true
eval_strategy: epoch
num_train_epochs: 10
seed: 0
warmup_epochs: 1.0
remove_unused_columns: false
push_to_hub: true
hub_model_id: Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
hub_strategy: end
hub_private_repo: false
method_args:
gamma: 0.125
alpha: 1.0
retain_loss_type: NLL
delta: 0.0
beta: 4.5
data:
forget:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${forget_split}
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
retain:
TOFU_QA_retain:
handler: QADataset
args:
hf_args:
name: ${retain_split}
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
anchor: forget
collator:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
eval:
tofu:
metrics:
forget_truth_ratio:
pre_compute:
forget_Q_A_PARA_Prob:
datasets:
TOFU_QA_forget_para:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
forget_Q_A_PERT_Prob:
datasets:
TOFU_QA_forget_pert:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: closer_to_1_better
forget_quality:
pre_compute:
forget_truth_ratio:
pre_compute:
forget_Q_A_PARA_Prob:
datasets:
TOFU_QA_forget_para:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
forget_Q_A_PERT_Prob:
datasets:
TOFU_QA_forget_pert:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: closer_to_1_better
access_key: forget
reference_logs:
retain_model_logs:
path: ${eval.tofu.retain_logs_path}
include:
forget_truth_ratio:
access_key: retain
handler: ks_test
forget_Q_A_Prob:
datasets:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
forget_Q_A_ROUGE:
datasets:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
model_utility:
pre_compute:
retain_Q_A_Prob:
datasets:
TOFU_QA_retain_eval:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
retain_Q_A_ROUGE:
datasets:
TOFU_QA_retain_eval:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
retain_Truth_Ratio:
pre_compute:
retain_Q_A_PARA_Prob:
datasets:
TOFU_QA_retain_para:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
retain_Q_A_PERT_Prob:
datasets:
TOFU_QA_retain_pert:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: true_better
ra_Q_A_Prob_normalised:
pre_compute:
ra_Q_A_Prob:
datasets:
TOFU_QA_ra:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
ra_Q_A_PERT_Prob:
datasets:
TOFU_QA_ra_pert:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: probability_w_options
ra_Q_A_ROUGE:
datasets:
TOFU_QA_ra:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
ra_Truth_Ratio:
pre_compute:
ra_Q_A_Prob:
datasets:
TOFU_QA_ra:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
ra_Q_A_PERT_Prob:
datasets:
TOFU_QA_ra_pert:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: true_better
wf_Q_A_Prob_normalised:
pre_compute:
wf_Q_A_Prob:
datasets:
TOFU_QA_wf:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
wf_Q_A_PERT_Prob:
datasets:
TOFU_QA_wf_pert:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: probability_w_options
wf_Q_A_ROUGE:
datasets:
TOFU_QA_wf:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
wf_Truth_Ratio:
pre_compute:
wf_Q_A_Prob:
datasets:
TOFU_QA_wf:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
wf_Q_A_PERT_Prob:
datasets:
TOFU_QA_wf_pert:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: true_better
handler: hm_aggregate
privleak:
pre_compute:
mia_min_k:
datasets:
TOFU_QA_forget:
access_key: forget
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
TOFU_QA_holdout:
access_key: holdout
handler: QADataset
args:
hf_args:
name: ${eval.tofu.holdout_split}
path: locuslab/TOFU
split: train
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
batch_size: ${eval.tofu.batch_size}
handler: mia_min_k
k: 0.4
access_key: forget
reference_logs:
retain_model_logs:
path: ${eval.tofu.retain_logs_path}
include:
mia_min_k:
access_key: retain
handler: privleak
ref_value: 0.5
extraction_strength:
datasets:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: extraction_strength
batch_size: ${eval.tofu.batch_size}
handler: TOFUEvaluator
output_dir: ${paths.output_dir}
overwrite: true
forget_split: ${forget_split}
holdout_split: ${holdout_split}
retain_logs_path: ${retain_logs_path}
question_key: ${question_key}
batch_size: 32
paths:
root_dir: .
data_dir: ${paths.root_dir}/data/
datasets: ${paths.root_dir}/configs/data/datasets
output_dir: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
work_dir: ${hydra:runtime.cwd}
forget_split: forget10
retain_split: retain90
holdout_split: holdout10
retain_logs_path: null
question_key: question
task_name: tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
mode: unlearn
quant:
enabled: true
scheme: int4_sym_per_channel
n_bits: 4
blocksize: 64
target_module_regex: model\.layers\..*\.(self_attn|mlp)\..*proj$
skip_module_regex: lm_head|embed_tokens

294
.hydra/hydra.yaml Normal file
View File

@@ -0,0 +1,294 @@
hydra:
run:
dir: ${paths.output_dir}
sweep:
dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
subdir: ${hydra.job.num}
launcher:
_target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
sweeper:
_target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
max_batch_size: null
params: null
help:
app_name: ${hydra.job.name}
header: '${hydra.help.app_name} is powered by Hydra.
'
footer: 'Powered by Hydra (https://hydra.cc)
Use --hydra-help to view Hydra specific help
'
template: '${hydra.help.header}
== Configuration groups ==
Compose your configuration from those groups (group=option)
$APP_CONFIG_GROUPS
== Config ==
Override anything in the config (foo.bar=value)
$CONFIG
${hydra.help.footer}
'
hydra_help:
template: 'Hydra (${hydra.runtime.version})
See https://hydra.cc for more info.
== Flags ==
$FLAGS_HELP
== Configuration groups ==
Compose your configuration from those groups (For example, append hydra/job_logging=disabled
to command line)
$HYDRA_CONFIG_GROUPS
Use ''--cfg hydra'' to Show the Hydra config.
'
hydra_help: ???
hydra_logging:
version: 1
formatters:
colorlog:
(): colorlog.ColoredFormatter
format: '[%(cyan)s%(asctime)s%(reset)s][%(purple)sHYDRA%(reset)s] %(message)s'
handlers:
console:
class: logging.StreamHandler
formatter: colorlog
stream: ext://sys.stdout
root:
level: INFO
handlers:
- console
disable_existing_loggers: false
job_logging:
version: 1
formatters:
simple:
format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
colorlog:
(): colorlog.ColoredFormatter
format: '[%(cyan)s%(asctime)s%(reset)s][%(blue)s%(name)s%(reset)s][%(log_color)s%(levelname)s%(reset)s]
- %(message)s'
log_colors:
DEBUG: purple
INFO: green
WARNING: yellow
ERROR: red
CRITICAL: red
handlers:
console:
class: logging.StreamHandler
formatter: colorlog
stream: ext://sys.stdout
file:
class: logging.FileHandler
formatter: simple
filename: ${hydra.runtime.output_dir}/${trainer.handler}.log
root:
level: INFO
handlers:
- console
- file
disable_existing_loggers: false
env: {}
mode: RUN
searchpath: []
callbacks: {}
output_subdir: .hydra
overrides:
hydra:
- hydra.mode=RUN
task:
- experiment=unlearn/tofu/default.yaml
- trainer=SimNPO
- task_name=tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
- model=Llama-3.2-1B-Instruct
- forget_split=forget10
- retain_split=retain90
- holdout_split=holdout10
- model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full
- model.model_args.attn_implementation=sdpa
- trainer.args.per_device_train_batch_size=4
- trainer.args.gradient_accumulation_steps=4
- trainer.args.ddp_find_unused_parameters=true
- trainer.args.do_eval=false
- trainer.args.save_strategy=no
- +trainer.args.push_to_hub=true
- +trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
- +trainer.args.hub_strategy=end
- +trainer.args.hub_private_repo=false
- paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
- +quant=qat_w4
job:
name: train
chdir: null
override_dirname: +quant=qat_w4,+trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4,+trainer.args.hub_private_repo=false,+trainer.args.hub_strategy=end,+trainer.args.push_to_hub=true,experiment=unlearn/tofu/default.yaml,forget_split=forget10,holdout_split=holdout10,model.model_args.attn_implementation=sdpa,model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full,model=Llama-3.2-1B-Instruct,paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4,retain_split=retain90,task_name=tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4,trainer.args.ddp_find_unused_parameters=true,trainer.args.do_eval=false,trainer.args.gradient_accumulation_steps=4,trainer.args.per_device_train_batch_size=4,trainer.args.save_strategy=no,trainer=SimNPO
id: ???
num: ???
config_name: unlearn.yaml
env_set: {}
env_copy: []
config:
override_dirname:
kv_sep: '='
item_sep: ','
exclude_keys: []
runtime:
version: 1.3.0
version_base: '1.3'
cwd: /home/yonsei_jong/open-unlearning
config_sources:
- path: hydra.conf
schema: pkg
provider: hydra
- path: /home/yonsei_jong/open-unlearning/configs
schema: file
provider: main
- path: hydra_plugins.hydra_colorlog.conf
schema: pkg
provider: hydra-colorlog
- path: ''
schema: structured
provider: schema
output_dir: /home/yonsei_jong/open-unlearning/saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
choices:
quant: qat_w4
experiment: unlearn/tofu/default.yaml
paths: default
hydra: default
eval: tofu
eval/tofu_metrics/../../collator@eval.tofu.metrics.extraction_strength.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.extraction_strength.datasets: TOFU_QA_forget
eval/tofu_metrics/.@eval.tofu.metrics.privleak.pre_compute.mia_min_k: mia_min_k
eval/tofu_metrics/./../../collator@eval.tofu.metrics.privleak.pre_compute.mia_min_k.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.privleak.pre_compute.mia_min_k.datasets: TOFU_MIA
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio: wf_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.datasets
: TOFU_QA_wf_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.datasets
: TOFU_QA_wf
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE: wf_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.datasets: TOFU_QA_wf
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised: wf_Q_A_Prob_normalised
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.datasets
: TOFU_QA_wf_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.datasets
: TOFU_QA_wf
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio: ra_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.datasets
: TOFU_QA_ra_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.datasets
: TOFU_QA_ra
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE: ra_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.datasets: TOFU_QA_ra
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised: ra_Q_A_Prob_normalised
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.datasets
: TOFU_QA_ra_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.datasets
: TOFU_QA_ra
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio: retain_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob: retain_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.datasets
: TOFU_QA_retain_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob: retain_Q_A_PARA_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.datasets
: TOFU_QA_retain_para
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE: retain_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.datasets: TOFU_QA_retain_eval
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob: retain_Q_A_Prob
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.datasets: TOFU_QA_retain_eval
eval/tofu_metrics/../../generation@eval.tofu.metrics.forget_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_ROUGE.datasets: TOFU_QA_forget
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_Prob.datasets: TOFU_QA_forget
eval/tofu_metrics/.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio: forget_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets
: TOFU_QA_forget_pert
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets
: TOFU_QA_forget_para
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets: TOFU_QA_forget_pert
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets: TOFU_QA_forget_para
collator: DataCollatorForSupervisedDataset
data: unlearn
data/datasets@data.eval: null
data/datasets@data.retain: TOFU_QA_retain
data/datasets@data.forget: TOFU_QA_forget
trainer: SimNPO
model: Llama-3.2-1B-Instruct
hydra/env: default
hydra/callbacks: null
hydra/job_logging: colorlog
hydra/hydra_logging: colorlog
hydra/hydra_help: default
hydra/help: default
hydra/sweeper: basic
hydra/launcher: basic
hydra/output: default
verbose: false

20
.hydra/overrides.yaml Normal file
View File

@@ -0,0 +1,20 @@
- experiment=unlearn/tofu/default.yaml
- trainer=SimNPO
- task_name=tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
- model=Llama-3.2-1B-Instruct
- forget_split=forget10
- retain_split=retain90
- holdout_split=holdout10
- model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full
- model.model_args.attn_implementation=sdpa
- trainer.args.per_device_train_batch_size=4
- trainer.args.gradient_accumulation_steps=4
- trainer.args.ddp_find_unused_parameters=true
- trainer.args.do_eval=false
- trainer.args.save_strategy=no
- +trainer.args.push_to_hub=true
- +trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
- +trainer.args.hub_strategy=end
- +trainer.args.hub_private_repo=false
- paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
- +quant=qat_w4

56
README.md Normal file
View File

@@ -0,0 +1,56 @@
---
library_name: transformers
license: bsd-3-clause
base_model: open-unlearning/tofu_Llama-3.2-1B-Instruct_full
tags:
- generated_from_trainer
model-index:
- name: tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
results: []
---
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
should probably proofread and complete it, then remove this comment. -->
# tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
This model is a fine-tuned version of [open-unlearning/tofu_Llama-3.2-1B-Instruct_full](https://huggingface.co/open-unlearning/tofu_Llama-3.2-1B-Instruct_full) on an unknown dataset.
## Model description
More information needed
## Intended uses & limitations
More information needed
## Training and evaluation data
More information needed
## Training procedure
### Training hyperparameters
The following hyperparameters were used during training:
- learning_rate: 1e-05
- train_batch_size: 4
- eval_batch_size: 16
- seed: 0
- gradient_accumulation_steps: 4
- total_train_batch_size: 16
- optimizer: Use OptimizerNames.PAGED_ADAMW with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
- lr_scheduler_type: linear
- lr_scheduler_warmup_steps: 25
- num_epochs: 10
### Training results
### Framework versions
- Transformers 4.51.3
- Pytorch 2.11.0+cu128
- Datasets 3.0.1
- Tokenizers 0.21.4

465
SimNPO.log Normal file
View File

@@ -0,0 +1,465 @@
[2026-05-21 13:41:49,828][model][INFO] - Setting pad_token as eos token: <|eot_id|>
[2026-05-21 13:41:53,824][evaluator][INFO] - Evaluations stored in the experiment directory: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
[2026-05-21 13:41:55,071][trainer][INFO] - SimNPO Trainer loaded, output_dir: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
[2026-05-21 13:41:55,642][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 13:41:55,642][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-0/evals/TOFU_EVAL.json
[2026-05-21 13:41:55,642][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-0/evals/TOFU_SUMMARY.json
[2026-05-21 13:41:57,435][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 13:42:14,138][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 13:42:49,130][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 13:42:49,131][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6448834990772232
[2026-05-21 13:42:49,137][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 13:42:49,138][metrics][INFO] - Evaluating forget_quality
[2026-05-21 13:42:49,138][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 13:42:49,138][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 13:42:51,141][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 13:42:55,954][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.10887884757248685
[2026-05-21 13:42:57,618][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 13:46:07,243][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.27890823225307393
[2026-05-21 13:46:08,886][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 13:46:16,399][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 13:47:36,147][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 13:47:43,718][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 13:48:10,569][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 13:48:12,248][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 13:48:15,546][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 13:48:19,756][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 13:48:21,599][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 13:48:58,918][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 13:48:58,918][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 13:48:58,918][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 13:49:00,580][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 13:49:02,793][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 13:49:06,973][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 13:49:08,385][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 13:50:05,059][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 13:50:05,060][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 13:50:05,060][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 13:50:05,060][metrics][INFO] - Evaluating model_utility
[2026-05-21 13:50:05,061][evaluator][INFO] - Result for metric model_utility: 0.2185219733034594
[2026-05-21 13:50:08,176][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 13:50:14,964][metrics][INFO] - Evaluating privleak
[2026-05-21 13:50:14,965][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 13:50:14,965][evaluator][INFO] - Result for metric privleak: -47.96999999040602
[2026-05-21 13:50:16,641][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 13:50:31,558][evaluator][INFO] - Result for metric extraction_strength: 0.05119585323242784
[2026-05-21 13:52:32,141][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 13:52:32,141][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-25/evals/TOFU_EVAL.json
[2026-05-21 13:52:32,141][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-25/evals/TOFU_SUMMARY.json
[2026-05-21 13:52:34,515][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 13:52:42,120][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 13:53:11,351][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 13:53:11,352][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6414198592494345
[2026-05-21 13:53:11,358][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 13:53:11,358][metrics][INFO] - Evaluating forget_quality
[2026-05-21 13:53:11,359][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 13:53:11,359][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 13:53:13,191][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 13:53:19,003][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.20349505057558417
[2026-05-21 13:53:20,753][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 13:54:33,238][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3363523902144183
[2026-05-21 13:54:35,217][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 13:54:42,714][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 13:55:40,635][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 13:55:45,320][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 13:56:01,893][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 13:56:03,568][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 13:56:05,655][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 13:56:08,019][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 13:56:09,276][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 13:56:57,945][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 13:56:57,946][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 13:56:57,946][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 13:56:59,881][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 13:57:01,754][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 13:57:03,621][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 13:57:04,874][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 13:57:10,920][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 13:57:10,921][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 13:57:10,921][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 13:57:10,921][metrics][INFO] - Evaluating model_utility
[2026-05-21 13:57:10,922][evaluator][INFO] - Result for metric model_utility: 0.27790674597563475
[2026-05-21 13:57:14,071][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 13:57:15,644][metrics][INFO] - Evaluating privleak
[2026-05-21 13:57:15,644][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 13:57:15,644][evaluator][INFO] - Result for metric privleak: -66.64374998667122
[2026-05-21 13:57:16,981][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 13:57:18,384][evaluator][INFO] - Result for metric extraction_strength: 0.062394619827353906
[2026-05-21 13:58:05,566][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 13:58:05,566][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-50/evals/TOFU_EVAL.json
[2026-05-21 13:58:05,566][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-50/evals/TOFU_SUMMARY.json
[2026-05-21 13:58:07,331][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 13:58:15,414][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 13:58:43,721][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 13:58:43,722][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6230931944181684
[2026-05-21 13:58:43,729][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 13:58:43,729][metrics][INFO] - Evaluating forget_quality
[2026-05-21 13:58:43,730][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 13:58:43,730][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 13:58:45,430][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 13:58:51,436][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.3349009375087917
[2026-05-21 13:58:53,116][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 14:00:00,893][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.38400245016174755
[2026-05-21 14:00:02,716][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 14:00:10,290][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 14:00:55,664][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 14:01:03,361][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 14:01:32,607][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 14:01:34,300][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 14:01:36,977][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 14:01:41,003][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 14:01:42,266][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 14:01:51,299][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 14:01:51,299][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:01:51,299][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 14:01:52,974][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 14:01:55,310][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 14:01:58,871][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 14:02:00,270][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 14:02:09,462][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 14:02:09,463][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:02:09,463][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 14:02:09,463][metrics][INFO] - Evaluating model_utility
[2026-05-21 14:02:09,464][evaluator][INFO] - Result for metric model_utility: 0.3793508809768731
[2026-05-21 14:02:12,459][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 14:02:18,002][metrics][INFO] - Evaluating privleak
[2026-05-21 14:02:18,002][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 14:02:18,002][evaluator][INFO] - Result for metric privleak: -82.57624998348474
[2026-05-21 14:02:19,846][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 14:02:32,706][evaluator][INFO] - Result for metric extraction_strength: 0.08291476394201429
[2026-05-21 14:03:43,858][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 14:03:43,858][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-75/evals/TOFU_EVAL.json
[2026-05-21 14:03:43,858][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-75/evals/TOFU_SUMMARY.json
[2026-05-21 14:03:45,532][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 14:03:50,772][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 14:04:07,357][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 14:04:07,358][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6189158593014839
[2026-05-21 14:04:07,365][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 14:04:07,365][metrics][INFO] - Evaluating forget_quality
[2026-05-21 14:04:07,365][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 14:04:07,365][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 14:04:09,123][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 14:04:12,470][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.34760695200413466
[2026-05-21 14:04:14,478][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 14:04:53,084][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.38394973457576287
[2026-05-21 14:04:54,774][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 14:05:02,239][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 14:05:30,694][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 14:05:36,590][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 14:05:59,909][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 14:06:01,526][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 14:06:03,492][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 14:06:07,836][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 14:06:09,116][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 14:06:18,131][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 14:06:18,131][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:06:18,131][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 14:06:19,811][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 14:06:22,281][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 14:06:25,957][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 14:06:27,284][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 14:06:34,497][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 14:06:34,498][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:06:34,498][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 14:06:34,498][metrics][INFO] - Evaluating model_utility
[2026-05-21 14:06:34,499][evaluator][INFO] - Result for metric model_utility: 0.38425409126783
[2026-05-21 14:06:38,348][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 14:06:43,812][metrics][INFO] - Evaluating privleak
[2026-05-21 14:06:43,813][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 14:06:43,813][evaluator][INFO] - Result for metric privleak: -83.71749998325649
[2026-05-21 14:06:45,548][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 14:07:08,688][evaluator][INFO] - Result for metric extraction_strength: 0.08463917597529186
[2026-05-21 14:08:21,983][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 14:08:21,984][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-100/evals/TOFU_EVAL.json
[2026-05-21 14:08:21,984][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-100/evals/TOFU_SUMMARY.json
[2026-05-21 14:08:23,767][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 14:08:31,957][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 14:09:01,576][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 14:09:01,577][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6181410546581807
[2026-05-21 14:09:01,583][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 14:09:01,584][metrics][INFO] - Evaluating forget_quality
[2026-05-21 14:09:01,584][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 14:09:01,584][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 14:09:03,337][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 14:09:09,301][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.35111760402098297
[2026-05-21 14:09:11,122][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 14:10:10,724][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3848115314453001
[2026-05-21 14:10:12,398][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 14:10:19,849][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 14:10:59,083][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 14:11:05,060][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 14:11:33,858][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 14:11:35,531][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 14:11:37,848][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 14:11:40,658][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 14:11:41,930][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 14:11:49,146][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 14:11:49,146][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:11:49,146][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 14:11:50,817][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 14:11:53,075][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 14:11:56,630][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 14:11:57,891][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 14:12:13,895][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 14:12:13,896][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:12:13,896][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 14:12:13,896][metrics][INFO] - Evaluating model_utility
[2026-05-21 14:12:13,897][evaluator][INFO] - Result for metric model_utility: 0.38945632253152707
[2026-05-21 14:12:17,302][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 14:12:18,792][metrics][INFO] - Evaluating privleak
[2026-05-21 14:12:18,792][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 14:12:18,792][evaluator][INFO] - Result for metric privleak: -83.93999998321199
[2026-05-21 14:12:20,070][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 14:12:21,679][evaluator][INFO] - Result for metric extraction_strength: 0.08717998187117214
[2026-05-21 14:13:07,731][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 14:13:07,731][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-125/evals/TOFU_EVAL.json
[2026-05-21 14:13:07,731][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-125/evals/TOFU_SUMMARY.json
[2026-05-21 14:13:09,455][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 14:13:15,520][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 14:13:44,789][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 14:13:44,790][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6182047623953388
[2026-05-21 14:13:44,796][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 14:13:44,797][metrics][INFO] - Evaluating forget_quality
[2026-05-21 14:13:44,797][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 14:13:44,797][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 14:13:46,438][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 14:13:51,259][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.3523250863514841
[2026-05-21 14:13:53,033][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 14:14:57,204][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3850742811515734
[2026-05-21 14:14:58,901][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 14:15:06,651][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 14:15:48,169][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 14:15:54,181][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 14:16:22,227][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 14:16:23,925][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 14:16:26,570][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 14:16:30,718][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 14:16:32,038][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 14:16:41,046][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 14:16:41,046][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:16:41,046][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 14:16:42,723][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 14:16:45,377][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 14:16:48,836][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 14:16:50,303][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 14:17:06,455][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 14:17:06,455][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:17:06,456][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 14:17:06,456][metrics][INFO] - Evaluating model_utility
[2026-05-21 14:17:06,456][evaluator][INFO] - Result for metric model_utility: 0.3867871595048101
[2026-05-21 14:17:09,811][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 14:17:15,247][metrics][INFO] - Evaluating privleak
[2026-05-21 14:17:15,248][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 14:17:15,248][evaluator][INFO] - Result for metric privleak: -84.0524999831895
[2026-05-21 14:17:17,040][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 14:17:32,756][evaluator][INFO] - Result for metric extraction_strength: 0.08658014531492801
[2026-05-21 14:18:42,024][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 14:18:42,024][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-150/evals/TOFU_EVAL.json
[2026-05-21 14:18:42,024][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-150/evals/TOFU_SUMMARY.json
[2026-05-21 14:18:43,899][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 14:18:51,499][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 14:19:17,007][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 14:19:17,008][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6177007290305743
[2026-05-21 14:19:17,014][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 14:19:17,015][metrics][INFO] - Evaluating forget_quality
[2026-05-21 14:19:17,015][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 14:19:17,015][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 14:19:18,823][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 14:19:24,824][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.35286230804398655
[2026-05-21 14:19:26,482][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 14:20:05,951][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.38548027127365925
[2026-05-21 14:20:07,665][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 14:20:14,399][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 14:20:46,571][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 14:20:54,476][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 14:21:21,302][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 14:21:23,506][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 14:21:26,158][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 14:21:30,200][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 14:21:31,450][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 14:21:39,581][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 14:21:39,582][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:21:39,582][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 14:21:41,267][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 14:21:43,599][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 14:21:47,062][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 14:21:48,317][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 14:22:04,276][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 14:22:04,276][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:22:04,276][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 14:22:04,276][metrics][INFO] - Evaluating model_utility
[2026-05-21 14:22:04,277][evaluator][INFO] - Result for metric model_utility: 0.389570021337874
[2026-05-21 14:22:07,214][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 14:22:12,747][metrics][INFO] - Evaluating privleak
[2026-05-21 14:22:12,747][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 14:22:12,747][evaluator][INFO] - Result for metric privleak: -84.09624998318074
[2026-05-21 14:22:14,458][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 14:22:37,635][evaluator][INFO] - Result for metric extraction_strength: 0.08737657392397029
[2026-05-21 14:23:50,677][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 14:23:50,678][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-175/evals/TOFU_EVAL.json
[2026-05-21 14:23:50,678][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-175/evals/TOFU_SUMMARY.json
[2026-05-21 14:23:52,366][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 14:24:00,068][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 14:24:29,732][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 14:24:29,733][evaluator][INFO] - Result for metric forget_truth_ratio: 0.61761660709215
[2026-05-21 14:24:29,740][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 14:24:29,740][metrics][INFO] - Evaluating forget_quality
[2026-05-21 14:24:29,741][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 14:24:29,741][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 14:24:31,434][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 14:24:37,439][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.3529359891824424
[2026-05-21 14:24:39,138][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 14:25:27,518][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3858334433873904
[2026-05-21 14:25:29,212][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 14:25:36,119][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 14:26:13,244][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 14:26:19,837][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 14:26:48,531][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 14:26:50,239][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 14:26:52,370][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 14:26:54,727][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 14:26:55,999][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 14:27:02,297][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 14:27:02,297][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:27:02,297][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 14:27:04,329][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 14:27:06,676][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 14:27:09,610][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 14:27:10,887][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 14:27:23,803][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 14:27:23,803][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:27:23,803][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 14:27:23,804][metrics][INFO] - Evaluating model_utility
[2026-05-21 14:27:23,804][evaluator][INFO] - Result for metric model_utility: 0.38596120600351796
[2026-05-21 14:27:26,809][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 14:27:32,599][metrics][INFO] - Evaluating privleak
[2026-05-21 14:27:32,600][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 14:27:32,600][evaluator][INFO] - Result for metric privleak: -84.109999983178
[2026-05-21 14:27:34,343][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 14:27:39,904][evaluator][INFO] - Result for metric extraction_strength: 0.08715686518263624
[2026-05-21 14:28:56,633][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 14:28:56,633][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-200/evals/TOFU_EVAL.json
[2026-05-21 14:28:56,633][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-200/evals/TOFU_SUMMARY.json
[2026-05-21 14:28:58,297][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 14:29:05,868][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 14:29:35,483][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 14:29:35,484][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6176789569373814
[2026-05-21 14:29:35,490][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 14:29:35,491][metrics][INFO] - Evaluating forget_quality
[2026-05-21 14:29:35,491][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 14:29:35,491][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 14:29:37,291][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 14:29:43,198][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.3530563661269844
[2026-05-21 14:29:44,887][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 14:30:35,000][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3858238347046325
[2026-05-21 14:30:36,945][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 14:30:44,482][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 14:31:29,528][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 14:31:37,866][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 14:32:04,499][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 14:32:06,162][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 14:32:08,832][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 14:32:13,184][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 14:32:14,438][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 14:32:21,953][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 14:32:21,953][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:32:21,953][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 14:32:23,774][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 14:32:25,562][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 14:32:28,749][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 14:32:30,166][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 14:32:46,413][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 14:32:46,414][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:32:46,414][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 14:32:46,414][metrics][INFO] - Evaluating model_utility
[2026-05-21 14:32:46,415][evaluator][INFO] - Result for metric model_utility: 0.38865403947163873
[2026-05-21 14:32:49,531][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 14:32:55,132][metrics][INFO] - Evaluating privleak
[2026-05-21 14:32:55,132][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 14:32:55,132][evaluator][INFO] - Result for metric privleak: -84.05874998318824
[2026-05-21 14:32:56,939][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 14:32:59,389][evaluator][INFO] - Result for metric extraction_strength: 0.0845172504995537
[2026-05-21 14:33:52,576][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 14:33:52,576][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-225/evals/TOFU_EVAL.json
[2026-05-21 14:33:52,576][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-225/evals/TOFU_SUMMARY.json
[2026-05-21 14:33:54,211][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 14:33:59,807][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 14:34:23,369][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 14:34:23,370][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6176135041820996
[2026-05-21 14:34:23,376][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 14:34:23,377][metrics][INFO] - Evaluating forget_quality
[2026-05-21 14:34:23,377][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 14:34:23,377][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 14:34:25,048][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 14:34:31,293][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.3531488827429712
[2026-05-21 14:34:38,330][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 14:35:32,093][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.38595172349337675
[2026-05-21 14:35:33,813][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 14:35:41,578][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 14:36:23,376][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 14:36:31,267][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 14:36:58,530][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 14:37:00,381][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 14:37:03,478][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 14:37:07,621][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 14:37:09,032][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 14:37:17,101][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 14:37:17,101][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:37:17,101][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 14:37:19,088][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 14:37:22,208][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 14:37:25,753][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 14:37:27,011][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 14:37:43,790][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 14:37:43,791][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:37:43,791][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 14:37:43,791][metrics][INFO] - Evaluating model_utility
[2026-05-21 14:37:43,792][evaluator][INFO] - Result for metric model_utility: 0.38656524448187823
[2026-05-21 14:37:46,811][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 14:37:52,207][metrics][INFO] - Evaluating privleak
[2026-05-21 14:37:52,207][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 14:37:52,207][evaluator][INFO] - Result for metric privleak: -84.14999998316999
[2026-05-21 14:37:54,588][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 14:38:16,613][evaluator][INFO] - Result for metric extraction_strength: 0.08690805565882671
[2026-05-21 14:39:24,024][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 14:39:24,024][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-250/evals/TOFU_EVAL.json
[2026-05-21 14:39:24,024][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-250/evals/TOFU_SUMMARY.json
[2026-05-21 14:39:25,713][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 14:39:33,358][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 14:39:58,593][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 14:39:58,594][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6175223364978973
[2026-05-21 14:39:58,600][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 14:39:58,601][metrics][INFO] - Evaluating forget_quality
[2026-05-21 14:39:58,601][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 14:39:58,601][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 14:40:00,392][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 14:40:06,399][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.3530087990872562
[2026-05-21 14:40:08,232][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 14:40:45,290][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3857650261516718
[2026-05-21 14:40:47,150][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 14:40:52,766][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 14:41:25,175][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 14:41:33,115][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 14:42:00,329][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 14:42:01,993][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 14:42:04,966][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 14:42:09,061][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 14:42:10,361][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 14:42:19,191][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 14:42:19,192][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:42:19,192][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 14:42:20,838][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 14:42:23,848][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 14:42:27,388][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 14:42:28,653][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 14:42:44,760][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 14:42:44,761][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 14:42:44,761][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 14:42:44,761][metrics][INFO] - Evaluating model_utility
[2026-05-21 14:42:44,762][evaluator][INFO] - Result for metric model_utility: 0.3884310780989943
[2026-05-21 14:42:48,251][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 14:42:53,673][metrics][INFO] - Evaluating privleak
[2026-05-21 14:42:53,674][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 14:42:53,674][evaluator][INFO] - Result for metric privleak: -84.10499998317901
[2026-05-21 14:42:55,653][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 14:43:17,396][evaluator][INFO] - Result for metric extraction_strength: 0.08670724689551795

39
config.json Normal file
View File

@@ -0,0 +1,39 @@
{
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 128000,
"eos_token_id": [
128001,
128008,
128009
],
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 2048,
"initializer_range": 0.02,
"intermediate_size": 8192,
"max_position_embeddings": 131072,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 16,
"num_key_value_heads": 8,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": {
"factor": 32.0,
"high_freq_factor": 4.0,
"low_freq_factor": 1.0,
"original_max_position_embeddings": 8192,
"rope_type": "llama3"
},
"rope_theta": 500000.0,
"tie_word_embeddings": true,
"torch_dtype": "bfloat16",
"transformers_version": "4.51.3",
"use_cache": true,
"vocab_size": 128256
}

12
generation_config.json Normal file
View File

@@ -0,0 +1,12 @@
{
"bos_token_id": 128000,
"do_sample": true,
"eos_token_id": [
128001,
128008,
128009
],
"temperature": 0.6,
"top_p": 0.9,
"transformers_version": "4.51.3"
}

3
logs/events.out.tfevents.1779338515.jeesup.2824533.0 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:678e7af1b8d1fbf7eeac34203aeff8f3dd88a8f002257566a3100e4ce55c8e78
size 20653

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:71d852a65c4c5330ae8269c3737245307f722eea0ec65a049fc6234ed86b3535
size 2471645608

17
special_tokens_map.json Normal file
View File

@@ -0,0 +1,17 @@
{
"bos_token": {
"content": "<|begin_of_text|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|eot_id|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": "<|eot_id|>"
}

BIN
tokenizer.json (Stored with Git LFS) Normal file
View File

Binary file not shown.

2064
tokenizer_config.json Normal file
View File

File diff suppressed because it is too large Load Diff

503
trainer_state.json Normal file
View File

@@ -0,0 +1,503 @@
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 10.0,
"eval_steps": 500,
"global_step": 250,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0,
"extraction_strength": 0.05119585323242784,
"forget_Q_A_Prob": 0.10887884757248685,
"forget_Q_A_ROUGE": 0.27890823225307393,
"forget_truth_ratio": 0.6448834990772232,
"model_utility": 0.2185219733034594,
"privleak": -47.96999999040602,
"step": 0
},
{
"epoch": 0.2,
"grad_norm": 660.0,
"learning_rate": 1.6000000000000001e-06,
"loss": 9.7906,
"step": 5
},
{
"epoch": 0.4,
"grad_norm": 580.0,
"learning_rate": 3.6000000000000003e-06,
"loss": 9.7614,
"step": 10
},
{
"epoch": 0.6,
"grad_norm": 592.0,
"learning_rate": 5.600000000000001e-06,
"loss": 9.9195,
"step": 15
},
{
"epoch": 0.8,
"grad_norm": 664.0,
"learning_rate": 7.600000000000001e-06,
"loss": 9.7584,
"step": 20
},
{
"epoch": 1.0,
"grad_norm": 426.0,
"learning_rate": 9.600000000000001e-06,
"loss": 8.8246,
"step": 25
},
{
"epoch": 1.0,
"extraction_strength": 0.062394619827353906,
"forget_Q_A_Prob": 0.20349505057558417,
"forget_Q_A_ROUGE": 0.3363523902144183,
"forget_truth_ratio": 0.6414198592494345,
"model_utility": 0.27790674597563475,
"privleak": -66.64374998667122,
"step": 25
},
{
"epoch": 1.2,
"grad_norm": 225.0,
"learning_rate": 9.822222222222223e-06,
"loss": 6.6271,
"step": 30
},
{
"epoch": 1.4,
"grad_norm": 155.0,
"learning_rate": 9.600000000000001e-06,
"loss": 5.4214,
"step": 35
},
{
"epoch": 1.6,
"grad_norm": 197.0,
"learning_rate": 9.377777777777779e-06,
"loss": 5.5142,
"step": 40
},
{
"epoch": 1.8,
"grad_norm": 134.0,
"learning_rate": 9.155555555555557e-06,
"loss": 4.6586,
"step": 45
},
{
"epoch": 2.0,
"grad_norm": 155.0,
"learning_rate": 8.933333333333333e-06,
"loss": 4.6746,
"step": 50
},
{
"epoch": 2.0,
"extraction_strength": 0.08291476394201429,
"forget_Q_A_Prob": 0.3349009375087917,
"forget_Q_A_ROUGE": 0.38400245016174755,
"forget_truth_ratio": 0.6230931944181684,
"model_utility": 0.3793508809768731,
"privleak": -82.57624998348474,
"step": 50
},
{
"epoch": 2.2,
"grad_norm": 139.0,
"learning_rate": 8.711111111111111e-06,
"loss": 4.7704,
"step": 55
},
{
"epoch": 2.4,
"grad_norm": 130.0,
"learning_rate": 8.48888888888889e-06,
"loss": 4.5076,
"step": 60
},
{
"epoch": 2.6,
"grad_norm": 150.0,
"learning_rate": 8.266666666666667e-06,
"loss": 4.6811,
"step": 65
},
{
"epoch": 2.8,
"grad_norm": 178.0,
"learning_rate": 8.044444444444444e-06,
"loss": 4.8506,
"step": 70
},
{
"epoch": 3.0,
"grad_norm": 143.0,
"learning_rate": 7.822222222222224e-06,
"loss": 4.5449,
"step": 75
},
{
"epoch": 3.0,
"extraction_strength": 0.08463917597529186,
"forget_Q_A_Prob": 0.34760695200413466,
"forget_Q_A_ROUGE": 0.38394973457576287,
"forget_truth_ratio": 0.6189158593014839,
"model_utility": 0.38425409126783,
"privleak": -83.71749998325649,
"step": 75
},
{
"epoch": 3.2,
"grad_norm": 139.0,
"learning_rate": 7.600000000000001e-06,
"loss": 4.6021,
"step": 80
},
{
"epoch": 3.4,
"grad_norm": 145.0,
"learning_rate": 7.377777777777778e-06,
"loss": 4.5443,
"step": 85
},
{
"epoch": 3.6,
"grad_norm": 164.0,
"learning_rate": 7.155555555555556e-06,
"loss": 4.4261,
"step": 90
},
{
"epoch": 3.8,
"grad_norm": 125.5,
"learning_rate": 6.9333333333333344e-06,
"loss": 4.6176,
"step": 95
},
{
"epoch": 4.0,
"grad_norm": 141.0,
"learning_rate": 6.711111111111111e-06,
"loss": 4.3231,
"step": 100
},
{
"epoch": 4.0,
"extraction_strength": 0.08717998187117214,
"forget_Q_A_Prob": 0.35111760402098297,
"forget_Q_A_ROUGE": 0.3848115314453001,
"forget_truth_ratio": 0.6181410546581807,
"model_utility": 0.38945632253152707,
"privleak": -83.93999998321199,
"step": 100
},
{
"epoch": 4.2,
"grad_norm": 134.0,
"learning_rate": 6.488888888888889e-06,
"loss": 4.4801,
"step": 105
},
{
"epoch": 4.4,
"grad_norm": 128.0,
"learning_rate": 6.266666666666668e-06,
"loss": 4.3562,
"step": 110
},
{
"epoch": 4.6,
"grad_norm": 142.0,
"learning_rate": 6.044444444444445e-06,
"loss": 4.4758,
"step": 115
},
{
"epoch": 4.8,
"grad_norm": 127.5,
"learning_rate": 5.822222222222223e-06,
"loss": 4.3376,
"step": 120
},
{
"epoch": 5.0,
"grad_norm": 131.0,
"learning_rate": 5.600000000000001e-06,
"loss": 4.3354,
"step": 125
},
{
"epoch": 5.0,
"extraction_strength": 0.08658014531492801,
"forget_Q_A_Prob": 0.3523250863514841,
"forget_Q_A_ROUGE": 0.3850742811515734,
"forget_truth_ratio": 0.6182047623953388,
"model_utility": 0.3867871595048101,
"privleak": -84.0524999831895,
"step": 125
},
{
"epoch": 5.2,
"grad_norm": 127.0,
"learning_rate": 5.3777777777777784e-06,
"loss": 4.415,
"step": 130
},
{
"epoch": 5.4,
"grad_norm": 119.0,
"learning_rate": 5.155555555555556e-06,
"loss": 4.3584,
"step": 135
},
{
"epoch": 5.6,
"grad_norm": 132.0,
"learning_rate": 4.933333333333334e-06,
"loss": 4.3697,
"step": 140
},
{
"epoch": 5.8,
"grad_norm": 163.0,
"learning_rate": 4.711111111111111e-06,
"loss": 4.3684,
"step": 145
},
{
"epoch": 6.0,
"grad_norm": 122.5,
"learning_rate": 4.488888888888889e-06,
"loss": 4.2881,
"step": 150
},
{
"epoch": 6.0,
"extraction_strength": 0.08737657392397029,
"forget_Q_A_Prob": 0.35286230804398655,
"forget_Q_A_ROUGE": 0.38548027127365925,
"forget_truth_ratio": 0.6177007290305743,
"model_utility": 0.389570021337874,
"privleak": -84.09624998318074,
"step": 150
},
{
"epoch": 6.2,
"grad_norm": 142.0,
"learning_rate": 4.266666666666668e-06,
"loss": 4.2205,
"step": 155
},
{
"epoch": 6.4,
"grad_norm": 142.0,
"learning_rate": 4.044444444444445e-06,
"loss": 4.3014,
"step": 160
},
{
"epoch": 6.6,
"grad_norm": 123.5,
"learning_rate": 3.8222222222222224e-06,
"loss": 4.3313,
"step": 165
},
{
"epoch": 6.8,
"grad_norm": 144.0,
"learning_rate": 3.6000000000000003e-06,
"loss": 4.4139,
"step": 170
},
{
"epoch": 7.0,
"grad_norm": 156.0,
"learning_rate": 3.377777777777778e-06,
"loss": 4.3248,
"step": 175
},
{
"epoch": 7.0,
"extraction_strength": 0.08715686518263624,
"forget_Q_A_Prob": 0.3529359891824424,
"forget_Q_A_ROUGE": 0.3858334433873904,
"forget_truth_ratio": 0.61761660709215,
"model_utility": 0.38596120600351796,
"privleak": -84.109999983178,
"step": 175
},
{
"epoch": 7.2,
"grad_norm": 128.0,
"learning_rate": 3.1555555555555555e-06,
"loss": 4.3152,
"step": 180
},
{
"epoch": 7.4,
"grad_norm": 120.0,
"learning_rate": 2.9333333333333338e-06,
"loss": 4.322,
"step": 185
},
{
"epoch": 7.6,
"grad_norm": 115.0,
"learning_rate": 2.7111111111111116e-06,
"loss": 4.3278,
"step": 190
},
{
"epoch": 7.8,
"grad_norm": 143.0,
"learning_rate": 2.488888888888889e-06,
"loss": 4.2159,
"step": 195
},
{
"epoch": 8.0,
"grad_norm": 132.0,
"learning_rate": 2.266666666666667e-06,
"loss": 4.2292,
"step": 200
},
{
"epoch": 8.0,
"extraction_strength": 0.0845172504995537,
"forget_Q_A_Prob": 0.3530563661269844,
"forget_Q_A_ROUGE": 0.3858238347046325,
"forget_truth_ratio": 0.6176789569373814,
"model_utility": 0.38865403947163873,
"privleak": -84.05874998318824,
"step": 200
},
{
"epoch": 8.2,
"grad_norm": 118.5,
"learning_rate": 2.0444444444444447e-06,
"loss": 4.1904,
"step": 205
},
{
"epoch": 8.4,
"grad_norm": 124.5,
"learning_rate": 1.8222222222222225e-06,
"loss": 4.223,
"step": 210
},
{
"epoch": 8.6,
"grad_norm": 143.0,
"learning_rate": 1.6000000000000001e-06,
"loss": 4.447,
"step": 215
},
{
"epoch": 8.8,
"grad_norm": 143.0,
"learning_rate": 1.377777777777778e-06,
"loss": 4.4513,
"step": 220
},
{
"epoch": 9.0,
"grad_norm": 126.0,
"learning_rate": 1.1555555555555556e-06,
"loss": 4.323,
"step": 225
},
{
"epoch": 9.0,
"extraction_strength": 0.08690805565882671,
"forget_Q_A_Prob": 0.3531488827429712,
"forget_Q_A_ROUGE": 0.38595172349337675,
"forget_truth_ratio": 0.6176135041820996,
"model_utility": 0.38656524448187823,
"privleak": -84.14999998316999,
"step": 225
},
{
"epoch": 9.2,
"grad_norm": 119.5,
"learning_rate": 9.333333333333334e-07,
"loss": 3.9312,
"step": 230
},
{
"epoch": 9.4,
"grad_norm": 134.0,
"learning_rate": 7.111111111111112e-07,
"loss": 4.3145,
"step": 235
},
{
"epoch": 9.6,
"grad_norm": 139.0,
"learning_rate": 4.88888888888889e-07,
"loss": 4.4723,
"step": 240
},
{
"epoch": 9.8,
"grad_norm": 120.5,
"learning_rate": 2.666666666666667e-07,
"loss": 4.4162,
"step": 245
},
{
"epoch": 10.0,
"grad_norm": 144.0,
"learning_rate": 4.444444444444445e-08,
"loss": 4.4783,
"step": 250
},
{
"epoch": 10.0,
"extraction_strength": 0.08670724689551795,
"forget_Q_A_Prob": 0.3530087990872562,
"forget_Q_A_ROUGE": 0.3857650261516718,
"forget_truth_ratio": 0.6175223364978973,
"model_utility": 0.3884310780989943,
"privleak": -84.10499998317901,
"step": 250
},
{
"epoch": 10.0,
"step": 250,
"total_flos": 0.0,
"train_loss": 5.017040328979492,
"train_runtime": 3681.7906,
"train_samples_per_second": 1.086,
"train_steps_per_second": 0.068
}
],
"logging_steps": 5,
"max_steps": 250,
"num_input_tokens_seen": 0,
"num_train_epochs": 10,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b0b82ceacc2b0032bbcdedfbcd7b04d9bad75e45e2bab41a6ac36d7b34330a15
size 5905