Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-06-04 16:06:16 +08:00
commit b66da29467
15 changed files with 4193 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.gitattributes vendored Normal file
View File

@@ -0,0 +1,36 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text

673
.hydra/config.yaml Normal file
View File

@@ -0,0 +1,673 @@
model:
model_args:
pretrained_model_name_or_path: open-unlearning/tofu_Llama-3.2-1B-Instruct_full
attn_implementation: sdpa
torch_dtype: bfloat16
tokenizer_args:
pretrained_model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
template_args:
apply_chat_template: true
system_prompt: You are a helpful assistant.
system_prompt_with_special_tokens: '<|begin_of_text|><|start_header_id|>system<|end_header_id|>
You are a helpful assistant.<|eot_id|>'
user_start_tag: '<|start_header_id|>user<|end_header_id|>
'
user_end_tag: <|eot_id|>
asst_start_tag: '<|start_header_id|>assistant<|end_header_id|>
'
asst_end_tag: <|eot_id|>
date_string: 10 Apr 2025
trainer:
handler: RMU
args:
per_device_train_batch_size: 4
per_device_eval_batch_size: 16
gradient_accumulation_steps: 4
learning_rate: 1.0e-05
bf16: true
bf16_full_eval: true
logging_steps: 5
output_dir: ${paths.output_dir}
logging_dir: ${trainer.args.output_dir}/logs
report_to: tensorboard
ddp_find_unused_parameters: true
gradient_checkpointing: false
optim: paged_adamw_32bit
save_strategy: 'no'
save_only_model: true
weight_decay: 0.01
do_train: true
do_eval: false
eval_on_start: true
eval_strategy: epoch
num_train_epochs: 10
seed: 0
warmup_epochs: 1.0
remove_unused_columns: false
push_to_hub: true
hub_model_id: Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
hub_strategy: end
hub_private_repo: false
method_args:
gamma: 1.0
alpha: 1
retain_loss_type: EMBED_DIFF
steering_coeff: 2
module_regex: model\.layers\.3
trainable_params_regex:
- .*
data:
forget:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${forget_split}
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
retain:
TOFU_QA_retain:
handler: QADataset
args:
hf_args:
name: ${retain_split}
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
anchor: forget
collator:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
eval:
tofu:
metrics:
forget_truth_ratio:
pre_compute:
forget_Q_A_PARA_Prob:
datasets:
TOFU_QA_forget_para:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
forget_Q_A_PERT_Prob:
datasets:
TOFU_QA_forget_pert:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: closer_to_1_better
forget_quality:
pre_compute:
forget_truth_ratio:
pre_compute:
forget_Q_A_PARA_Prob:
datasets:
TOFU_QA_forget_para:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
forget_Q_A_PERT_Prob:
datasets:
TOFU_QA_forget_pert:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: closer_to_1_better
access_key: forget
reference_logs:
retain_model_logs:
path: ${eval.tofu.retain_logs_path}
include:
forget_truth_ratio:
access_key: retain
handler: ks_test
forget_Q_A_Prob:
datasets:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
forget_Q_A_ROUGE:
datasets:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
model_utility:
pre_compute:
retain_Q_A_Prob:
datasets:
TOFU_QA_retain_eval:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
retain_Q_A_ROUGE:
datasets:
TOFU_QA_retain_eval:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
retain_Truth_Ratio:
pre_compute:
retain_Q_A_PARA_Prob:
datasets:
TOFU_QA_retain_para:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: paraphrased_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
retain_Q_A_PERT_Prob:
datasets:
TOFU_QA_retain_pert:
handler: QADataset
args:
hf_args:
name: retain_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: true_better
ra_Q_A_Prob_normalised:
pre_compute:
ra_Q_A_Prob:
datasets:
TOFU_QA_ra:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
ra_Q_A_PERT_Prob:
datasets:
TOFU_QA_ra_pert:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: probability_w_options
ra_Q_A_ROUGE:
datasets:
TOFU_QA_ra:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
ra_Truth_Ratio:
pre_compute:
ra_Q_A_Prob:
datasets:
TOFU_QA_ra:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
ra_Q_A_PERT_Prob:
datasets:
TOFU_QA_ra_pert:
handler: QADataset
args:
hf_args:
name: real_authors_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: true_better
wf_Q_A_Prob_normalised:
pre_compute:
wf_Q_A_Prob:
datasets:
TOFU_QA_wf:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
wf_Q_A_PERT_Prob:
datasets:
TOFU_QA_wf_pert:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: probability_w_options
wf_Q_A_ROUGE:
datasets:
TOFU_QA_wf:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
predict_with_generate: true
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: left
index: index
generation_args:
do_sample: false
top_p: null
temperature: null
max_new_tokens: 200
use_cache: true
handler: rouge
rouge_type: rougeL_recall
batch_size: ${eval.tofu.batch_size}
wf_Truth_Ratio:
pre_compute:
wf_Q_A_Prob:
datasets:
TOFU_QA_wf:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: correct
wf_Q_A_PERT_Prob:
datasets:
TOFU_QA_wf_pert:
handler: QADataset
args:
hf_args:
name: world_facts_perturbed
split: train
path: locuslab/TOFU
question_key: question
answer_key: perturbed_answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: probability
batch_size: ${eval.tofu.batch_size}
access_key: wrong
handler: truth_ratio
aggregator: true_better
handler: hm_aggregate
privleak:
pre_compute:
mia_min_k:
datasets:
TOFU_QA_forget:
access_key: forget
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
TOFU_QA_holdout:
access_key: holdout
handler: QADataset
args:
hf_args:
name: ${eval.tofu.holdout_split}
path: locuslab/TOFU
split: train
question_key: question
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
batch_size: ${eval.tofu.batch_size}
handler: mia_min_k
k: 0.4
access_key: forget
reference_logs:
retain_model_logs:
path: ${eval.tofu.retain_logs_path}
include:
mia_min_k:
access_key: retain
handler: privleak
ref_value: 0.5
extraction_strength:
datasets:
TOFU_QA_forget:
handler: QADataset
args:
hf_args:
name: ${eval.tofu.forget_split}_perturbed
split: train
path: locuslab/TOFU
question_key: ${eval.tofu.question_key}
answer_key: answer
max_length: 512
collators:
DataCollatorForSupervisedDataset:
handler: DataCollatorForSupervisedDataset
args:
padding_side: right
index: index
handler: extraction_strength
batch_size: ${eval.tofu.batch_size}
handler: TOFUEvaluator
output_dir: ${paths.output_dir}
overwrite: true
forget_split: ${forget_split}
holdout_split: ${holdout_split}
retain_logs_path: ${retain_logs_path}
question_key: ${question_key}
batch_size: 32
paths:
root_dir: .
data_dir: ${paths.root_dir}/data/
datasets: ${paths.root_dir}/configs/data/datasets
output_dir: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
work_dir: ${hydra:runtime.cwd}
forget_split: forget10
retain_split: retain90
holdout_split: holdout10
retain_logs_path: null
question_key: question
task_name: tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
mode: unlearn
quant:
enabled: true
scheme: int4_sym_per_channel
n_bits: 4
blocksize: 64
target_module_regex: model\.layers\..*\.(self_attn|mlp)\..*proj$
skip_module_regex: lm_head|embed_tokens

295
.hydra/hydra.yaml Normal file
View File

@@ -0,0 +1,295 @@
hydra:
run:
dir: ${paths.output_dir}
sweep:
dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
subdir: ${hydra.job.num}
launcher:
_target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
sweeper:
_target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
max_batch_size: null
params: null
help:
app_name: ${hydra.job.name}
header: '${hydra.help.app_name} is powered by Hydra.
'
footer: 'Powered by Hydra (https://hydra.cc)
Use --hydra-help to view Hydra specific help
'
template: '${hydra.help.header}
== Configuration groups ==
Compose your configuration from those groups (group=option)
$APP_CONFIG_GROUPS
== Config ==
Override anything in the config (foo.bar=value)
$CONFIG
${hydra.help.footer}
'
hydra_help:
template: 'Hydra (${hydra.runtime.version})
See https://hydra.cc for more info.
== Flags ==
$FLAGS_HELP
== Configuration groups ==
Compose your configuration from those groups (For example, append hydra/job_logging=disabled
to command line)
$HYDRA_CONFIG_GROUPS
Use ''--cfg hydra'' to Show the Hydra config.
'
hydra_help: ???
hydra_logging:
version: 1
formatters:
colorlog:
(): colorlog.ColoredFormatter
format: '[%(cyan)s%(asctime)s%(reset)s][%(purple)sHYDRA%(reset)s] %(message)s'
handlers:
console:
class: logging.StreamHandler
formatter: colorlog
stream: ext://sys.stdout
root:
level: INFO
handlers:
- console
disable_existing_loggers: false
job_logging:
version: 1
formatters:
simple:
format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
colorlog:
(): colorlog.ColoredFormatter
format: '[%(cyan)s%(asctime)s%(reset)s][%(blue)s%(name)s%(reset)s][%(log_color)s%(levelname)s%(reset)s]
- %(message)s'
log_colors:
DEBUG: purple
INFO: green
WARNING: yellow
ERROR: red
CRITICAL: red
handlers:
console:
class: logging.StreamHandler
formatter: colorlog
stream: ext://sys.stdout
file:
class: logging.FileHandler
formatter: simple
filename: ${hydra.runtime.output_dir}/${trainer.handler}.log
root:
level: INFO
handlers:
- console
- file
disable_existing_loggers: false
env: {}
mode: RUN
searchpath: []
callbacks: {}
output_subdir: .hydra
overrides:
hydra:
- hydra.mode=RUN
task:
- experiment=unlearn/tofu/default.yaml
- trainer=RMU
- task_name=tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
- model=Llama-3.2-1B-Instruct
- forget_split=forget10
- retain_split=retain90
- holdout_split=holdout10
- model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full
- model.model_args.attn_implementation=sdpa
- trainer.args.per_device_train_batch_size=4
- trainer.args.gradient_accumulation_steps=4
- trainer.args.ddp_find_unused_parameters=true
- trainer.args.do_eval=false
- trainer.args.save_strategy=no
- +trainer.args.push_to_hub=true
- +trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
- +trainer.args.hub_strategy=end
- +trainer.args.hub_private_repo=false
- paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
- +quant=qat_w4
- trainer.method_args.module_regex=model\.layers\.3
job:
name: train
chdir: null
override_dirname: +quant=qat_w4,+trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4,+trainer.args.hub_private_repo=false,+trainer.args.hub_strategy=end,+trainer.args.push_to_hub=true,experiment=unlearn/tofu/default.yaml,forget_split=forget10,holdout_split=holdout10,model.model_args.attn_implementation=sdpa,model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full,model=Llama-3.2-1B-Instruct,paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4,retain_split=retain90,task_name=tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4,trainer.args.ddp_find_unused_parameters=true,trainer.args.do_eval=false,trainer.args.gradient_accumulation_steps=4,trainer.args.per_device_train_batch_size=4,trainer.args.save_strategy=no,trainer.method_args.module_regex=model\.layers\.3,trainer=RMU
id: ???
num: ???
config_name: unlearn.yaml
env_set: {}
env_copy: []
config:
override_dirname:
kv_sep: '='
item_sep: ','
exclude_keys: []
runtime:
version: 1.3.0
version_base: '1.3'
cwd: /home/yonsei_jong/open-unlearning
config_sources:
- path: hydra.conf
schema: pkg
provider: hydra
- path: /home/yonsei_jong/open-unlearning/configs
schema: file
provider: main
- path: hydra_plugins.hydra_colorlog.conf
schema: pkg
provider: hydra-colorlog
- path: ''
schema: structured
provider: schema
output_dir: /home/yonsei_jong/open-unlearning/saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
choices:
quant: qat_w4
experiment: unlearn/tofu/default.yaml
paths: default
hydra: default
eval: tofu
eval/tofu_metrics/../../collator@eval.tofu.metrics.extraction_strength.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.extraction_strength.datasets: TOFU_QA_forget
eval/tofu_metrics/.@eval.tofu.metrics.privleak.pre_compute.mia_min_k: mia_min_k
eval/tofu_metrics/./../../collator@eval.tofu.metrics.privleak.pre_compute.mia_min_k.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.privleak.pre_compute.mia_min_k.datasets: TOFU_MIA
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio: wf_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.datasets
: TOFU_QA_wf_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.datasets
: TOFU_QA_wf
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE: wf_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.datasets: TOFU_QA_wf
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised: wf_Q_A_Prob_normalised
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.datasets
: TOFU_QA_wf_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.datasets
: TOFU_QA_wf
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio: ra_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.datasets
: TOFU_QA_ra_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.datasets
: TOFU_QA_ra
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE: ra_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.datasets: TOFU_QA_ra
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised: ra_Q_A_Prob_normalised
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.datasets
: TOFU_QA_ra_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.datasets
: TOFU_QA_ra
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio: retain_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob: retain_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.datasets
: TOFU_QA_retain_pert
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob: retain_Q_A_PARA_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.datasets
: TOFU_QA_retain_para
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE: retain_Q_A_ROUGE
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.datasets: TOFU_QA_retain_eval
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob: retain_Q_A_Prob
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.datasets: TOFU_QA_retain_eval
eval/tofu_metrics/../../generation@eval.tofu.metrics.forget_Q_A_ROUGE.generation_args: default
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_ROUGE.datasets: TOFU_QA_forget
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_Prob.datasets: TOFU_QA_forget
eval/tofu_metrics/.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio: forget_Truth_Ratio
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets
: TOFU_QA_forget_pert
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators
: DataCollatorForSupervisedDatasetwithIndex
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets
: TOFU_QA_forget_para
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets: TOFU_QA_forget_pert
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets: TOFU_QA_forget_para
collator: DataCollatorForSupervisedDataset
data: unlearn
data/datasets@data.eval: null
data/datasets@data.retain: TOFU_QA_retain
data/datasets@data.forget: TOFU_QA_forget
trainer: RMU
model: Llama-3.2-1B-Instruct
hydra/env: default
hydra/callbacks: null
hydra/job_logging: colorlog
hydra/hydra_logging: colorlog
hydra/hydra_help: default
hydra/help: default
hydra/sweeper: basic
hydra/launcher: basic
hydra/output: default
verbose: false

21
.hydra/overrides.yaml Normal file
View File

@@ -0,0 +1,21 @@
- experiment=unlearn/tofu/default.yaml
- trainer=RMU
- task_name=tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
- model=Llama-3.2-1B-Instruct
- forget_split=forget10
- retain_split=retain90
- holdout_split=holdout10
- model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full
- model.model_args.attn_implementation=sdpa
- trainer.args.per_device_train_batch_size=4
- trainer.args.gradient_accumulation_steps=4
- trainer.args.ddp_find_unused_parameters=true
- trainer.args.do_eval=false
- trainer.args.save_strategy=no
- +trainer.args.push_to_hub=true
- +trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
- +trainer.args.hub_strategy=end
- +trainer.args.hub_private_repo=false
- paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
- +quant=qat_w4
- trainer.method_args.module_regex=model\.layers\.3

56
README.md Normal file
View File

@@ -0,0 +1,56 @@
---
library_name: transformers
license: bsd-3-clause
base_model: open-unlearning/tofu_Llama-3.2-1B-Instruct_full
tags:
- generated_from_trainer
model-index:
- name: tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
results: []
---
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
should probably proofread and complete it, then remove this comment. -->
# tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
This model is a fine-tuned version of [open-unlearning/tofu_Llama-3.2-1B-Instruct_full](https://huggingface.co/open-unlearning/tofu_Llama-3.2-1B-Instruct_full) on an unknown dataset.
## Model description
More information needed
## Intended uses & limitations
More information needed
## Training and evaluation data
More information needed
## Training procedure
### Training hyperparameters
The following hyperparameters were used during training:
- learning_rate: 1e-05
- train_batch_size: 4
- eval_batch_size: 16
- seed: 0
- gradient_accumulation_steps: 4
- total_train_batch_size: 16
- optimizer: Use OptimizerNames.PAGED_ADAMW with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
- lr_scheduler_type: linear
- lr_scheduler_warmup_steps: 25
- num_epochs: 10
### Training results
### Framework versions
- Transformers 4.51.3
- Pytorch 2.11.0+cu128
- Datasets 3.0.1
- Tokenizers 0.21.4

465
RMU.log Normal file
View File

@@ -0,0 +1,465 @@
[2026-05-21 11:57:33,698][model][INFO] - Setting pad_token as eos token: <|eot_id|>
[2026-05-21 11:57:37,813][evaluator][INFO] - Evaluations stored in the experiment directory: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
[2026-05-21 11:57:39,435][trainer][INFO] - RMU Trainer loaded, output_dir: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
[2026-05-21 11:57:40,023][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 11:57:40,023][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-0/evals/TOFU_EVAL.json
[2026-05-21 11:57:40,023][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-0/evals/TOFU_SUMMARY.json
[2026-05-21 11:57:41,506][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 11:57:58,264][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 11:58:38,873][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 11:58:38,874][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6448834990772232
[2026-05-21 11:58:38,880][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 11:58:38,881][metrics][INFO] - Evaluating forget_quality
[2026-05-21 11:58:38,881][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 11:58:38,881][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 11:58:40,563][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 11:58:46,744][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.10887884757248685
[2026-05-21 11:58:48,505][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 12:02:17,635][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.27890823225307393
[2026-05-21 12:02:19,330][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 12:02:27,753][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 12:04:33,587][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 12:04:41,251][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 12:05:12,905][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 12:05:14,652][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 12:05:18,181][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 12:05:21,757][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 12:05:23,000][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 12:06:07,138][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 12:06:07,139][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:06:07,139][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 12:06:08,822][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 12:06:11,152][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 12:06:15,810][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 12:06:17,088][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 12:07:14,539][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 12:07:14,539][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:07:14,539][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 12:07:14,540][metrics][INFO] - Evaluating model_utility
[2026-05-21 12:07:14,544][evaluator][INFO] - Result for metric model_utility: 0.2185219733034594
[2026-05-21 12:07:17,574][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 12:07:21,214][metrics][INFO] - Evaluating privleak
[2026-05-21 12:07:21,214][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 12:07:21,214][evaluator][INFO] - Result for metric privleak: -47.96999999040602
[2026-05-21 12:07:22,668][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 12:07:23,965][evaluator][INFO] - Result for metric extraction_strength: 0.05119585323242784
[2026-05-21 12:08:53,751][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 12:08:53,751][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-25/evals/TOFU_EVAL.json
[2026-05-21 12:08:53,751][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-25/evals/TOFU_SUMMARY.json
[2026-05-21 12:08:55,728][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 12:09:02,060][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 12:09:26,356][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 12:09:26,357][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6614321776157142
[2026-05-21 12:09:26,364][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 12:09:26,364][metrics][INFO] - Evaluating forget_quality
[2026-05-21 12:09:26,364][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 12:09:26,364][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 12:09:28,002][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 12:09:33,162][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.044040033797500655
[2026-05-21 12:09:34,852][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 12:11:29,404][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.19825707891118127
[2026-05-21 12:11:31,096][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 12:11:39,019][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 12:13:45,155][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 12:13:52,720][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 12:14:21,986][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 12:14:23,693][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 12:14:26,349][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 12:14:30,389][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 12:14:31,749][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 12:15:44,420][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 12:15:44,421][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:15:44,421][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 12:15:46,154][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 12:15:48,537][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 12:15:51,980][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 12:15:53,337][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 12:16:22,460][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 12:16:22,460][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:16:22,460][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 12:16:22,461][metrics][INFO] - Evaluating model_utility
[2026-05-21 12:16:22,461][evaluator][INFO] - Result for metric model_utility: 0.08605862105946929
[2026-05-21 12:16:25,761][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 12:16:31,351][metrics][INFO] - Evaluating privleak
[2026-05-21 12:16:31,351][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 12:16:31,351][evaluator][INFO] - Result for metric privleak: -22.12749999557451
[2026-05-21 12:16:33,075][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 12:16:42,339][evaluator][INFO] - Result for metric extraction_strength: 0.04014118794649353
[2026-05-21 12:17:06,543][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 12:17:06,543][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-50/evals/TOFU_EVAL.json
[2026-05-21 12:17:06,544][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-50/evals/TOFU_SUMMARY.json
[2026-05-21 12:17:08,209][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 12:17:16,386][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 12:17:44,453][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 12:17:44,454][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6646114327873448
[2026-05-21 12:17:44,460][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 12:17:44,461][metrics][INFO] - Evaluating forget_quality
[2026-05-21 12:17:44,461][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 12:17:44,461][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 12:17:46,123][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 12:17:51,886][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.028171376134705498
[2026-05-21 12:17:53,595][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 12:19:56,169][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.18136216120754584
[2026-05-21 12:19:58,075][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 12:20:05,461][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 12:22:11,777][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 12:22:19,434][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 12:22:48,615][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 12:22:50,385][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 12:22:53,191][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 12:22:57,341][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 12:22:58,995][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 12:23:38,005][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 12:23:38,005][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:23:38,005][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 12:23:39,646][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 12:23:42,040][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 12:23:45,003][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 12:23:46,257][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 12:24:23,368][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 12:24:23,369][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:24:23,369][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 12:24:23,369][metrics][INFO] - Evaluating model_utility
[2026-05-21 12:24:23,370][evaluator][INFO] - Result for metric model_utility: 0.07766678967490327
[2026-05-21 12:24:26,397][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 12:24:31,922][metrics][INFO] - Evaluating privleak
[2026-05-21 12:24:31,923][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 12:24:31,923][evaluator][INFO] - Result for metric privleak: -16.837499996632495
[2026-05-21 12:24:33,630][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 12:24:42,580][evaluator][INFO] - Result for metric extraction_strength: 0.03555271291935598
[2026-05-21 12:25:09,643][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 12:25:09,643][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-75/evals/TOFU_EVAL.json
[2026-05-21 12:25:09,643][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-75/evals/TOFU_SUMMARY.json
[2026-05-21 12:25:11,296][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 12:25:19,068][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 12:25:36,970][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 12:25:36,971][evaluator][INFO] - Result for metric forget_truth_ratio: 0.666956633608444
[2026-05-21 12:25:36,977][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 12:25:36,978][metrics][INFO] - Evaluating forget_quality
[2026-05-21 12:25:36,978][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 12:25:36,978][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 12:25:38,613][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 12:25:44,827][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.025599835189350415
[2026-05-21 12:25:46,467][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 12:27:48,567][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.17665323136017774
[2026-05-21 12:27:50,319][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 12:27:58,030][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 12:30:05,340][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 12:30:12,963][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 12:30:39,143][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 12:30:40,835][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 12:30:43,488][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 12:30:47,643][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 12:30:48,950][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 12:31:28,106][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 12:31:28,107][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:31:28,107][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 12:31:29,859][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 12:31:32,220][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 12:31:35,651][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 12:31:36,902][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 12:32:16,255][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 12:32:16,256][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:32:16,256][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 12:32:16,256][metrics][INFO] - Evaluating model_utility
[2026-05-21 12:32:16,257][evaluator][INFO] - Result for metric model_utility: 0.025169849004668194
[2026-05-21 12:32:19,201][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 12:32:24,841][metrics][INFO] - Evaluating privleak
[2026-05-21 12:32:24,841][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 12:32:24,841][evaluator][INFO] - Result for metric privleak: -15.60999999687799
[2026-05-21 12:32:26,598][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 12:32:30,837][evaluator][INFO] - Result for metric extraction_strength: 0.03542379317218397
[2026-05-21 12:32:55,301][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 12:32:55,301][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-100/evals/TOFU_EVAL.json
[2026-05-21 12:32:55,301][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-100/evals/TOFU_SUMMARY.json
[2026-05-21 12:32:56,927][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 12:33:02,940][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 12:33:32,151][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 12:33:32,152][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6650891487506957
[2026-05-21 12:33:32,158][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 12:33:32,159][metrics][INFO] - Evaluating forget_quality
[2026-05-21 12:33:32,159][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 12:33:32,159][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 12:33:33,843][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 12:33:38,385][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.024368888099124887
[2026-05-21 12:33:39,662][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 12:35:08,076][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.17340090189465512
[2026-05-21 12:35:09,710][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 12:35:14,668][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 12:37:19,876][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 12:37:27,741][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 12:37:55,967][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 12:37:57,746][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 12:38:00,481][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 12:38:04,623][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 12:38:06,064][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 12:38:40,084][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 12:38:40,084][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:38:40,084][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 12:38:41,711][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 12:38:44,062][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 12:38:47,631][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 12:38:48,878][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 12:39:28,314][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 12:39:28,314][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:39:28,314][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 12:39:28,315][metrics][INFO] - Evaluating model_utility
[2026-05-21 12:39:28,315][evaluator][INFO] - Result for metric model_utility: 0.07572111776600261
[2026-05-21 12:39:31,238][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 12:39:36,722][metrics][INFO] - Evaluating privleak
[2026-05-21 12:39:36,722][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 12:39:36,722][evaluator][INFO] - Result for metric privleak: -15.054999996988997
[2026-05-21 12:39:38,473][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 12:39:45,286][evaluator][INFO] - Result for metric extraction_strength: 0.034881082155764
[2026-05-21 12:40:09,613][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 12:40:09,613][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-125/evals/TOFU_EVAL.json
[2026-05-21 12:40:09,613][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-125/evals/TOFU_SUMMARY.json
[2026-05-21 12:40:11,550][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 12:40:19,424][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 12:40:49,178][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 12:40:49,179][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6653264816436281
[2026-05-21 12:40:49,185][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 12:40:49,186][metrics][INFO] - Evaluating forget_quality
[2026-05-21 12:40:49,186][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 12:40:49,186][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 12:40:50,857][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 12:40:56,868][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.023090821445148322
[2026-05-21 12:40:58,563][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 12:42:59,770][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.17290647721627245
[2026-05-21 12:43:01,413][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 12:43:08,531][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 12:44:48,920][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 12:44:56,881][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 12:45:26,437][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 12:45:28,234][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 12:45:31,003][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 12:45:35,107][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 12:45:36,368][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 12:46:14,962][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 12:46:14,962][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:46:14,962][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 12:46:16,815][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 12:46:19,204][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 12:46:21,563][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 12:46:22,945][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 12:46:59,910][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 12:46:59,911][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:46:59,911][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 12:46:59,911][metrics][INFO] - Evaluating model_utility
[2026-05-21 12:46:59,912][evaluator][INFO] - Result for metric model_utility: 0.061282254233379956
[2026-05-21 12:47:02,933][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 12:47:08,271][metrics][INFO] - Evaluating privleak
[2026-05-21 12:47:08,271][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 12:47:08,271][evaluator][INFO] - Result for metric privleak: -14.518749997096252
[2026-05-21 12:47:10,069][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 12:47:32,061][evaluator][INFO] - Result for metric extraction_strength: 0.03492969775585019
[2026-05-21 12:47:52,836][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 12:47:52,836][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-150/evals/TOFU_EVAL.json
[2026-05-21 12:47:52,836][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-150/evals/TOFU_SUMMARY.json
[2026-05-21 12:47:54,493][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 12:48:02,146][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 12:48:31,769][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 12:48:31,770][evaluator][INFO] - Result for metric forget_truth_ratio: 0.665408632731608
[2026-05-21 12:48:31,776][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 12:48:31,777][metrics][INFO] - Evaluating forget_quality
[2026-05-21 12:48:31,777][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 12:48:31,777][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 12:48:33,422][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 12:48:37,862][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.022637412079275235
[2026-05-21 12:48:39,116][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 12:50:46,079][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.16969368902404242
[2026-05-21 12:50:48,146][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 12:50:54,741][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 12:52:56,349][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 12:53:02,778][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 12:53:20,514][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 12:53:22,173][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 12:53:25,211][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 12:53:29,602][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 12:53:31,058][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 12:54:09,744][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 12:54:09,745][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:54:09,745][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 12:54:11,393][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 12:54:13,787][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 12:54:17,260][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 12:54:18,762][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 12:54:58,987][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 12:54:58,988][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 12:54:58,988][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 12:54:58,988][metrics][INFO] - Evaluating model_utility
[2026-05-21 12:54:58,989][evaluator][INFO] - Result for metric model_utility: 0.07293884758887383
[2026-05-21 12:55:01,964][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 12:55:04,458][metrics][INFO] - Evaluating privleak
[2026-05-21 12:55:04,458][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 12:55:04,458][evaluator][INFO] - Result for metric privleak: -14.298749997140254
[2026-05-21 12:55:06,059][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 12:55:09,527][evaluator][INFO] - Result for metric extraction_strength: 0.034710650136802565
[2026-05-21 12:55:31,076][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 12:55:31,076][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-175/evals/TOFU_EVAL.json
[2026-05-21 12:55:31,076][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-175/evals/TOFU_SUMMARY.json
[2026-05-21 12:55:32,704][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 12:55:38,755][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 12:56:06,854][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 12:56:06,855][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6661571899518759
[2026-05-21 12:56:06,861][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 12:56:06,862][metrics][INFO] - Evaluating forget_quality
[2026-05-21 12:56:06,862][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 12:56:06,862][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 12:56:08,508][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 12:56:14,367][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.022689154595791478
[2026-05-21 12:56:16,040][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 12:58:20,768][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.17071292253126477
[2026-05-21 12:58:22,468][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 12:58:29,886][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 13:00:37,081][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 13:00:43,094][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 13:01:10,854][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 13:01:12,568][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 13:01:15,228][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 13:01:19,302][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 13:01:20,611][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 13:01:54,745][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 13:01:54,745][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 13:01:54,745][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 13:01:56,526][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 13:01:58,924][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 13:02:02,375][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 13:02:03,624][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 13:02:27,784][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 13:02:27,784][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 13:02:27,784][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 13:02:27,784][metrics][INFO] - Evaluating model_utility
[2026-05-21 13:02:27,785][evaluator][INFO] - Result for metric model_utility: 0.024938334488339972
[2026-05-21 13:02:30,834][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 13:02:35,356][metrics][INFO] - Evaluating privleak
[2026-05-21 13:02:35,356][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 13:02:35,356][evaluator][INFO] - Result for metric privleak: -14.50124999709975
[2026-05-21 13:02:36,768][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 13:02:52,494][evaluator][INFO] - Result for metric extraction_strength: 0.03454297434412678
[2026-05-21 13:03:16,128][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 13:03:16,129][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-200/evals/TOFU_EVAL.json
[2026-05-21 13:03:16,129][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-200/evals/TOFU_SUMMARY.json
[2026-05-21 13:03:17,801][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 13:03:25,631][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 13:03:56,057][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 13:03:56,058][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6657672475603854
[2026-05-21 13:03:56,064][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 13:03:56,065][metrics][INFO] - Evaluating forget_quality
[2026-05-21 13:03:56,065][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 13:03:56,065][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 13:03:57,750][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 13:04:02,208][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.022478839480972967
[2026-05-21 13:04:03,476][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 13:06:08,123][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.17469059451335844
[2026-05-21 13:06:09,777][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 13:06:17,186][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 13:08:23,471][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 13:08:31,153][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 13:09:00,265][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 13:09:01,944][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 13:09:04,573][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 13:09:07,999][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 13:09:09,674][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 13:09:45,248][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 13:09:45,249][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 13:09:45,249][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 13:09:46,946][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 13:09:49,056][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 13:09:52,602][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 13:09:54,048][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 13:10:30,231][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 13:10:30,231][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 13:10:30,231][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 13:10:30,232][metrics][INFO] - Evaluating model_utility
[2026-05-21 13:10:30,232][evaluator][INFO] - Result for metric model_utility: 0.02489956345780598
[2026-05-21 13:10:33,282][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 13:10:36,848][metrics][INFO] - Evaluating privleak
[2026-05-21 13:10:36,849][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 13:10:36,849][evaluator][INFO] - Result for metric privleak: -13.848749997230259
[2026-05-21 13:10:38,234][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 13:10:56,184][evaluator][INFO] - Result for metric extraction_strength: 0.03481065013680257
[2026-05-21 13:11:08,088][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 13:11:08,088][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-225/evals/TOFU_EVAL.json
[2026-05-21 13:11:08,088][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-225/evals/TOFU_SUMMARY.json
[2026-05-21 13:11:09,829][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 13:11:17,751][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 13:11:46,542][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 13:11:46,543][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6661828793220076
[2026-05-21 13:11:46,550][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 13:11:46,550][metrics][INFO] - Evaluating forget_quality
[2026-05-21 13:11:46,550][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 13:11:46,550][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 13:11:48,331][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 13:11:54,570][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.02249195015472651
[2026-05-21 13:11:56,226][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 13:13:55,029][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.16997095669432055
[2026-05-21 13:13:56,746][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 13:14:04,157][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 13:16:10,552][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 13:16:18,183][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 13:16:47,385][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 13:16:49,063][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 13:16:51,612][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 13:16:55,772][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 13:16:57,083][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 13:17:36,254][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 13:17:36,254][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 13:17:36,254][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 13:17:37,889][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 13:17:40,234][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 13:17:43,663][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 13:17:45,073][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 13:18:21,334][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 13:18:21,334][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 13:18:21,334][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 13:18:21,335][metrics][INFO] - Evaluating model_utility
[2026-05-21 13:18:21,335][evaluator][INFO] - Result for metric model_utility: 0.034445471917355024
[2026-05-21 13:18:24,581][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 13:18:30,118][metrics][INFO] - Evaluating privleak
[2026-05-21 13:18:30,118][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 13:18:30,118][evaluator][INFO] - Result for metric privleak: -13.99624999720075
[2026-05-21 13:18:31,791][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 13:18:33,441][evaluator][INFO] - Result for metric extraction_strength: 0.03500545533160776
[2026-05-21 13:18:48,997][evaluator][INFO] - ***** Running TOFU evaluation suite *****
[2026-05-21 13:18:48,998][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-250/evals/TOFU_EVAL.json
[2026-05-21 13:18:48,998][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-250/evals/TOFU_SUMMARY.json
[2026-05-21 13:18:50,692][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
[2026-05-21 13:18:58,986][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
[2026-05-21 13:19:25,257][metrics][INFO] - Evaluating forget_truth_ratio
[2026-05-21 13:19:25,258][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6682139209275343
[2026-05-21 13:19:25,264][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
[2026-05-21 13:19:25,265][metrics][INFO] - Evaluating forget_quality
[2026-05-21 13:19:25,265][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
[2026-05-21 13:19:25,265][evaluator][INFO] - Result for metric forget_quality: None
[2026-05-21 13:19:26,973][metrics][INFO] - Evaluating forget_Q_A_Prob
[2026-05-21 13:19:32,982][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.022312334740927326
[2026-05-21 13:19:34,673][metrics][INFO] - Evaluating forget_Q_A_ROUGE
[2026-05-21 13:21:16,645][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.16821955009936282
[2026-05-21 13:21:18,356][metrics][INFO] - Evaluating retain_Q_A_Prob
[2026-05-21 13:21:25,924][metrics][INFO] - Evaluating retain_Q_A_ROUGE
[2026-05-21 13:23:30,694][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
[2026-05-21 13:23:38,159][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
[2026-05-21 13:24:07,357][metrics][INFO] - Evaluating retain_Truth_Ratio
[2026-05-21 13:24:09,157][metrics][INFO] - Evaluating ra_Q_A_Prob
[2026-05-21 13:24:11,661][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
[2026-05-21 13:24:15,807][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
[2026-05-21 13:24:17,063][metrics][INFO] - Evaluating ra_Q_A_ROUGE
[2026-05-21 13:24:56,249][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
[2026-05-21 13:24:56,250][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
[2026-05-21 13:24:56,250][metrics][INFO] - Evaluating ra_Truth_Ratio
[2026-05-21 13:24:57,929][metrics][INFO] - Evaluating wf_Q_A_Prob
[2026-05-21 13:25:00,268][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
[2026-05-21 13:25:03,792][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
[2026-05-21 13:25:05,243][metrics][INFO] - Evaluating wf_Q_A_ROUGE
[2026-05-21 13:25:41,021][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
[2026-05-21 13:25:41,021][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
[2026-05-21 13:25:41,021][metrics][INFO] - Evaluating wf_Truth_Ratio
[2026-05-21 13:25:41,021][metrics][INFO] - Evaluating model_utility
[2026-05-21 13:25:41,022][evaluator][INFO] - Result for metric model_utility: 0.024945798930326533
[2026-05-21 13:25:44,127][metrics][INFO] - Evaluating mia_min_k
[2026-05-21 13:25:49,445][metrics][INFO] - Evaluating privleak
[2026-05-21 13:25:49,445][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
[2026-05-21 13:25:49,445][evaluator][INFO] - Result for metric privleak: -14.171249997165742
[2026-05-21 13:25:52,550][metrics][INFO] - Evaluating extraction_strength
[2026-05-21 13:26:14,618][evaluator][INFO] - Result for metric extraction_strength: 0.034667360093512525

39
config.json Normal file
View File

@@ -0,0 +1,39 @@
{
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 128000,
"eos_token_id": [
128001,
128008,
128009
],
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 2048,
"initializer_range": 0.02,
"intermediate_size": 8192,
"max_position_embeddings": 131072,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 16,
"num_key_value_heads": 8,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": {
"factor": 32.0,
"high_freq_factor": 4.0,
"low_freq_factor": 1.0,
"original_max_position_embeddings": 8192,
"rope_type": "llama3"
},
"rope_theta": 500000.0,
"tie_word_embeddings": true,
"torch_dtype": "bfloat16",
"transformers_version": "4.51.3",
"use_cache": true,
"vocab_size": 128256
}

12
generation_config.json Normal file
View File

@@ -0,0 +1,12 @@
{
"bos_token_id": 128000,
"do_sample": true,
"eos_token_id": [
128001,
128008,
128009
],
"temperature": 0.6,
"top_p": 0.9,
"transformers_version": "4.51.3"
}

3
logs/events.out.tfevents.1779332260.jeesup.2790209.0 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:e890fdd40ced5857881b87c9686bd84699393fe0b5adc51c418731d10f800f12
size 20641

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9fd74f8f3d05bbe00721eec97719405626cf0b66353316d9214c03bf26635df1
size 2471645608

17
special_tokens_map.json Normal file
View File

@@ -0,0 +1,17 @@
{
"bos_token": {
"content": "<|begin_of_text|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|eot_id|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": "<|eot_id|>"
}

BIN
tokenizer.json (Stored with Git LFS) Normal file
View File

Binary file not shown.

2064
tokenizer_config.json Normal file
View File

File diff suppressed because it is too large Load Diff

503
trainer_state.json Normal file
View File

@@ -0,0 +1,503 @@
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 10.0,
"eval_steps": 500,
"global_step": 250,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0,
"extraction_strength": 0.05119585323242784,
"forget_Q_A_Prob": 0.10887884757248685,
"forget_Q_A_ROUGE": 0.27890823225307393,
"forget_truth_ratio": 0.6448834990772232,
"model_utility": 0.2185219733034594,
"privleak": -47.96999999040602,
"step": 0
},
{
"epoch": 0.2,
"grad_norm": 0.33203125,
"learning_rate": 1.6000000000000001e-06,
"loss": 0.0302,
"step": 5
},
{
"epoch": 0.4,
"grad_norm": 0.31640625,
"learning_rate": 3.6000000000000003e-06,
"loss": 0.0295,
"step": 10
},
{
"epoch": 0.6,
"grad_norm": 0.2890625,
"learning_rate": 5.600000000000001e-06,
"loss": 0.0286,
"step": 15
},
{
"epoch": 0.8,
"grad_norm": 0.3203125,
"learning_rate": 7.600000000000001e-06,
"loss": 0.0293,
"step": 20
},
{
"epoch": 1.0,
"grad_norm": 0.1494140625,
"learning_rate": 9.600000000000001e-06,
"loss": 0.0271,
"step": 25
},
{
"epoch": 1.0,
"extraction_strength": 0.04014118794649353,
"forget_Q_A_Prob": 0.044040033797500655,
"forget_Q_A_ROUGE": 0.19825707891118127,
"forget_truth_ratio": 0.6614321776157142,
"model_utility": 0.08605862105946929,
"privleak": -22.12749999557451,
"step": 25
},
{
"epoch": 1.2,
"grad_norm": 0.11083984375,
"learning_rate": 9.822222222222223e-06,
"loss": 0.0254,
"step": 30
},
{
"epoch": 1.4,
"grad_norm": 0.10400390625,
"learning_rate": 9.600000000000001e-06,
"loss": 0.0254,
"step": 35
},
{
"epoch": 1.6,
"grad_norm": 0.08984375,
"learning_rate": 9.377777777777779e-06,
"loss": 0.024,
"step": 40
},
{
"epoch": 1.8,
"grad_norm": 0.10205078125,
"learning_rate": 9.155555555555557e-06,
"loss": 0.0246,
"step": 45
},
{
"epoch": 2.0,
"grad_norm": 0.08544921875,
"learning_rate": 8.933333333333333e-06,
"loss": 0.0241,
"step": 50
},
{
"epoch": 2.0,
"extraction_strength": 0.03555271291935598,
"forget_Q_A_Prob": 0.028171376134705498,
"forget_Q_A_ROUGE": 0.18136216120754584,
"forget_truth_ratio": 0.6646114327873448,
"model_utility": 0.07766678967490327,
"privleak": -16.837499996632495,
"step": 50
},
{
"epoch": 2.2,
"grad_norm": 0.087890625,
"learning_rate": 8.711111111111111e-06,
"loss": 0.0242,
"step": 55
},
{
"epoch": 2.4,
"grad_norm": 0.10595703125,
"learning_rate": 8.48888888888889e-06,
"loss": 0.0243,
"step": 60
},
{
"epoch": 2.6,
"grad_norm": 0.09033203125,
"learning_rate": 8.266666666666667e-06,
"loss": 0.024,
"step": 65
},
{
"epoch": 2.8,
"grad_norm": 0.0859375,
"learning_rate": 8.044444444444444e-06,
"loss": 0.0236,
"step": 70
},
{
"epoch": 3.0,
"grad_norm": 0.0791015625,
"learning_rate": 7.822222222222224e-06,
"loss": 0.0228,
"step": 75
},
{
"epoch": 3.0,
"extraction_strength": 0.03542379317218397,
"forget_Q_A_Prob": 0.025599835189350415,
"forget_Q_A_ROUGE": 0.17665323136017774,
"forget_truth_ratio": 0.666956633608444,
"model_utility": 0.025169849004668194,
"privleak": -15.60999999687799,
"step": 75
},
{
"epoch": 3.2,
"grad_norm": 0.083984375,
"learning_rate": 7.600000000000001e-06,
"loss": 0.023,
"step": 80
},
{
"epoch": 3.4,
"grad_norm": 0.07958984375,
"learning_rate": 7.377777777777778e-06,
"loss": 0.0228,
"step": 85
},
{
"epoch": 3.6,
"grad_norm": 0.08544921875,
"learning_rate": 7.155555555555556e-06,
"loss": 0.0236,
"step": 90
},
{
"epoch": 3.8,
"grad_norm": 0.07763671875,
"learning_rate": 6.9333333333333344e-06,
"loss": 0.0235,
"step": 95
},
{
"epoch": 4.0,
"grad_norm": 0.08251953125,
"learning_rate": 6.711111111111111e-06,
"loss": 0.0233,
"step": 100
},
{
"epoch": 4.0,
"extraction_strength": 0.034881082155764,
"forget_Q_A_Prob": 0.024368888099124887,
"forget_Q_A_ROUGE": 0.17340090189465512,
"forget_truth_ratio": 0.6650891487506957,
"model_utility": 0.07572111776600261,
"privleak": -15.054999996988997,
"step": 100
},
{
"epoch": 4.2,
"grad_norm": 0.08544921875,
"learning_rate": 6.488888888888889e-06,
"loss": 0.0243,
"step": 105
},
{
"epoch": 4.4,
"grad_norm": 0.08154296875,
"learning_rate": 6.266666666666668e-06,
"loss": 0.0231,
"step": 110
},
{
"epoch": 4.6,
"grad_norm": 0.08056640625,
"learning_rate": 6.044444444444445e-06,
"loss": 0.0231,
"step": 115
},
{
"epoch": 4.8,
"grad_norm": 0.080078125,
"learning_rate": 5.822222222222223e-06,
"loss": 0.0231,
"step": 120
},
{
"epoch": 5.0,
"grad_norm": 0.0810546875,
"learning_rate": 5.600000000000001e-06,
"loss": 0.0225,
"step": 125
},
{
"epoch": 5.0,
"extraction_strength": 0.03492969775585019,
"forget_Q_A_Prob": 0.023090821445148322,
"forget_Q_A_ROUGE": 0.17290647721627245,
"forget_truth_ratio": 0.6653264816436281,
"model_utility": 0.061282254233379956,
"privleak": -14.518749997096252,
"step": 125
},
{
"epoch": 5.2,
"grad_norm": 0.0810546875,
"learning_rate": 5.3777777777777784e-06,
"loss": 0.0235,
"step": 130
},
{
"epoch": 5.4,
"grad_norm": 0.08203125,
"learning_rate": 5.155555555555556e-06,
"loss": 0.0227,
"step": 135
},
{
"epoch": 5.6,
"grad_norm": 0.09619140625,
"learning_rate": 4.933333333333334e-06,
"loss": 0.0235,
"step": 140
},
{
"epoch": 5.8,
"grad_norm": 0.0947265625,
"learning_rate": 4.711111111111111e-06,
"loss": 0.0245,
"step": 145
},
{
"epoch": 6.0,
"grad_norm": 0.0849609375,
"learning_rate": 4.488888888888889e-06,
"loss": 0.0225,
"step": 150
},
{
"epoch": 6.0,
"extraction_strength": 0.034710650136802565,
"forget_Q_A_Prob": 0.022637412079275235,
"forget_Q_A_ROUGE": 0.16969368902404242,
"forget_truth_ratio": 0.665408632731608,
"model_utility": 0.07293884758887383,
"privleak": -14.298749997140254,
"step": 150
},
{
"epoch": 6.2,
"grad_norm": 0.07861328125,
"learning_rate": 4.266666666666668e-06,
"loss": 0.0239,
"step": 155
},
{
"epoch": 6.4,
"grad_norm": 0.08447265625,
"learning_rate": 4.044444444444445e-06,
"loss": 0.0234,
"step": 160
},
{
"epoch": 6.6,
"grad_norm": 0.0986328125,
"learning_rate": 3.8222222222222224e-06,
"loss": 0.0236,
"step": 165
},
{
"epoch": 6.8,
"grad_norm": 0.091796875,
"learning_rate": 3.6000000000000003e-06,
"loss": 0.023,
"step": 170
},
{
"epoch": 7.0,
"grad_norm": 0.0791015625,
"learning_rate": 3.377777777777778e-06,
"loss": 0.0231,
"step": 175
},
{
"epoch": 7.0,
"extraction_strength": 0.03454297434412678,
"forget_Q_A_Prob": 0.022689154595791478,
"forget_Q_A_ROUGE": 0.17071292253126477,
"forget_truth_ratio": 0.6661571899518759,
"model_utility": 0.024938334488339972,
"privleak": -14.50124999709975,
"step": 175
},
{
"epoch": 7.2,
"grad_norm": 0.0859375,
"learning_rate": 3.1555555555555555e-06,
"loss": 0.0234,
"step": 180
},
{
"epoch": 7.4,
"grad_norm": 0.083984375,
"learning_rate": 2.9333333333333338e-06,
"loss": 0.0229,
"step": 185
},
{
"epoch": 7.6,
"grad_norm": 0.0849609375,
"learning_rate": 2.7111111111111116e-06,
"loss": 0.0231,
"step": 190
},
{
"epoch": 7.8,
"grad_norm": 0.080078125,
"learning_rate": 2.488888888888889e-06,
"loss": 0.0234,
"step": 195
},
{
"epoch": 8.0,
"grad_norm": 0.08056640625,
"learning_rate": 2.266666666666667e-06,
"loss": 0.023,
"step": 200
},
{
"epoch": 8.0,
"extraction_strength": 0.03481065013680257,
"forget_Q_A_Prob": 0.022478839480972967,
"forget_Q_A_ROUGE": 0.17469059451335844,
"forget_truth_ratio": 0.6657672475603854,
"model_utility": 0.02489956345780598,
"privleak": -13.848749997230259,
"step": 200
},
{
"epoch": 8.2,
"grad_norm": 0.08251953125,
"learning_rate": 2.0444444444444447e-06,
"loss": 0.0234,
"step": 205
},
{
"epoch": 8.4,
"grad_norm": 0.0771484375,
"learning_rate": 1.8222222222222225e-06,
"loss": 0.0243,
"step": 210
},
{
"epoch": 8.6,
"grad_norm": 0.0869140625,
"learning_rate": 1.6000000000000001e-06,
"loss": 0.0231,
"step": 215
},
{
"epoch": 8.8,
"grad_norm": 0.080078125,
"learning_rate": 1.377777777777778e-06,
"loss": 0.0227,
"step": 220
},
{
"epoch": 9.0,
"grad_norm": 0.07177734375,
"learning_rate": 1.1555555555555556e-06,
"loss": 0.0229,
"step": 225
},
{
"epoch": 9.0,
"extraction_strength": 0.03500545533160776,
"forget_Q_A_Prob": 0.02249195015472651,
"forget_Q_A_ROUGE": 0.16997095669432055,
"forget_truth_ratio": 0.6661828793220076,
"model_utility": 0.034445471917355024,
"privleak": -13.99624999720075,
"step": 225
},
{
"epoch": 9.2,
"grad_norm": 0.076171875,
"learning_rate": 9.333333333333334e-07,
"loss": 0.0223,
"step": 230
},
{
"epoch": 9.4,
"grad_norm": 0.0791015625,
"learning_rate": 7.111111111111112e-07,
"loss": 0.0232,
"step": 235
},
{
"epoch": 9.6,
"grad_norm": 0.0849609375,
"learning_rate": 4.88888888888889e-07,
"loss": 0.023,
"step": 240
},
{
"epoch": 9.8,
"grad_norm": 0.0849609375,
"learning_rate": 2.666666666666667e-07,
"loss": 0.0235,
"step": 245
},
{
"epoch": 10.0,
"grad_norm": 0.0830078125,
"learning_rate": 4.444444444444445e-08,
"loss": 0.0232,
"step": 250
},
{
"epoch": 10.0,
"extraction_strength": 0.034667360093512525,
"forget_Q_A_Prob": 0.022312334740927326,
"forget_Q_A_ROUGE": 0.16821955009936282,
"forget_truth_ratio": 0.6682139209275343,
"model_utility": 0.024945798930326533,
"privleak": -14.171249997165742,
"step": 250
},
{
"epoch": 10.0,
"step": 250,
"total_flos": 0.0,
"train_loss": 0.0240108642578125,
"train_runtime": 5314.6353,
"train_samples_per_second": 0.753,
"train_steps_per_second": 0.047
}
],
"logging_steps": 5,
"max_steps": 250,
"num_input_tokens_seen": 0,
"num_train_epochs": 10,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:21cd2890c569286b2ba5b3dca58e45827bdb891c3f9981b3dfbc87551e9551ce
size 5841