初始化项目，由ModelHub XC社区提供模型
Model: Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4 Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||||
671
.hydra/config.yaml
Normal file
671
.hydra/config.yaml
Normal file
@@ -0,0 +1,671 @@
|
|||||||
|
model:
|
||||||
|
model_args:
|
||||||
|
pretrained_model_name_or_path: open-unlearning/tofu_Llama-3.2-1B-Instruct_full
|
||||||
|
attn_implementation: sdpa
|
||||||
|
torch_dtype: bfloat16
|
||||||
|
tokenizer_args:
|
||||||
|
pretrained_model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
|
||||||
|
template_args:
|
||||||
|
apply_chat_template: true
|
||||||
|
system_prompt: You are a helpful assistant.
|
||||||
|
system_prompt_with_special_tokens: '<|begin_of_text|><|start_header_id|>system<|end_header_id|>
|
||||||
|
|
||||||
|
|
||||||
|
You are a helpful assistant.<|eot_id|>'
|
||||||
|
user_start_tag: '<|start_header_id|>user<|end_header_id|>
|
||||||
|
|
||||||
|
|
||||||
|
'
|
||||||
|
user_end_tag: <|eot_id|>
|
||||||
|
asst_start_tag: '<|start_header_id|>assistant<|end_header_id|>
|
||||||
|
|
||||||
|
|
||||||
|
'
|
||||||
|
asst_end_tag: <|eot_id|>
|
||||||
|
date_string: 10 Apr 2025
|
||||||
|
trainer:
|
||||||
|
handler: SimNPO
|
||||||
|
args:
|
||||||
|
per_device_train_batch_size: 4
|
||||||
|
per_device_eval_batch_size: 16
|
||||||
|
gradient_accumulation_steps: 4
|
||||||
|
learning_rate: 1.0e-05
|
||||||
|
bf16: true
|
||||||
|
bf16_full_eval: true
|
||||||
|
logging_steps: 5
|
||||||
|
output_dir: ${paths.output_dir}
|
||||||
|
logging_dir: ${trainer.args.output_dir}/logs
|
||||||
|
report_to: tensorboard
|
||||||
|
ddp_find_unused_parameters: true
|
||||||
|
gradient_checkpointing: false
|
||||||
|
optim: paged_adamw_32bit
|
||||||
|
save_strategy: 'no'
|
||||||
|
save_only_model: true
|
||||||
|
weight_decay: 0.01
|
||||||
|
do_train: true
|
||||||
|
do_eval: false
|
||||||
|
eval_on_start: true
|
||||||
|
eval_strategy: epoch
|
||||||
|
num_train_epochs: 10
|
||||||
|
seed: 0
|
||||||
|
warmup_epochs: 1.0
|
||||||
|
remove_unused_columns: false
|
||||||
|
push_to_hub: true
|
||||||
|
hub_model_id: Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
hub_strategy: end
|
||||||
|
hub_private_repo: false
|
||||||
|
method_args:
|
||||||
|
gamma: 0.125
|
||||||
|
alpha: 1.0
|
||||||
|
retain_loss_type: NLL
|
||||||
|
delta: 0.0
|
||||||
|
beta: 4.5
|
||||||
|
data:
|
||||||
|
forget:
|
||||||
|
TOFU_QA_forget:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${forget_split}
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
retain:
|
||||||
|
TOFU_QA_retain:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${retain_split}
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
anchor: forget
|
||||||
|
collator:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
eval:
|
||||||
|
tofu:
|
||||||
|
metrics:
|
||||||
|
forget_truth_ratio:
|
||||||
|
pre_compute:
|
||||||
|
forget_Q_A_PARA_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget_para:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: paraphrased_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
forget_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: truth_ratio
|
||||||
|
aggregator: closer_to_1_better
|
||||||
|
forget_quality:
|
||||||
|
pre_compute:
|
||||||
|
forget_truth_ratio:
|
||||||
|
pre_compute:
|
||||||
|
forget_Q_A_PARA_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget_para:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: paraphrased_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
forget_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: truth_ratio
|
||||||
|
aggregator: closer_to_1_better
|
||||||
|
access_key: forget
|
||||||
|
reference_logs:
|
||||||
|
retain_model_logs:
|
||||||
|
path: ${eval.tofu.retain_logs_path}
|
||||||
|
include:
|
||||||
|
forget_truth_ratio:
|
||||||
|
access_key: retain
|
||||||
|
handler: ks_test
|
||||||
|
forget_Q_A_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
forget_Q_A_ROUGE:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
predict_with_generate: true
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: left
|
||||||
|
index: index
|
||||||
|
generation_args:
|
||||||
|
do_sample: false
|
||||||
|
top_p: null
|
||||||
|
temperature: null
|
||||||
|
max_new_tokens: 200
|
||||||
|
use_cache: true
|
||||||
|
handler: rouge
|
||||||
|
rouge_type: rougeL_recall
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
model_utility:
|
||||||
|
pre_compute:
|
||||||
|
retain_Q_A_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_retain_eval:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: retain_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
retain_Q_A_ROUGE:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_retain_eval:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: retain_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
predict_with_generate: true
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: left
|
||||||
|
index: index
|
||||||
|
generation_args:
|
||||||
|
do_sample: false
|
||||||
|
top_p: null
|
||||||
|
temperature: null
|
||||||
|
max_new_tokens: 200
|
||||||
|
use_cache: true
|
||||||
|
handler: rouge
|
||||||
|
rouge_type: rougeL_recall
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
retain_Truth_Ratio:
|
||||||
|
pre_compute:
|
||||||
|
retain_Q_A_PARA_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_retain_para:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: retain_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: paraphrased_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
retain_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_retain_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: retain_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: truth_ratio
|
||||||
|
aggregator: true_better
|
||||||
|
ra_Q_A_Prob_normalised:
|
||||||
|
pre_compute:
|
||||||
|
ra_Q_A_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_ra:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: real_authors_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
ra_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_ra_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: real_authors_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: probability_w_options
|
||||||
|
ra_Q_A_ROUGE:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_ra:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: real_authors_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
predict_with_generate: true
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: left
|
||||||
|
index: index
|
||||||
|
generation_args:
|
||||||
|
do_sample: false
|
||||||
|
top_p: null
|
||||||
|
temperature: null
|
||||||
|
max_new_tokens: 200
|
||||||
|
use_cache: true
|
||||||
|
handler: rouge
|
||||||
|
rouge_type: rougeL_recall
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
ra_Truth_Ratio:
|
||||||
|
pre_compute:
|
||||||
|
ra_Q_A_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_ra:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: real_authors_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
ra_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_ra_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: real_authors_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: truth_ratio
|
||||||
|
aggregator: true_better
|
||||||
|
wf_Q_A_Prob_normalised:
|
||||||
|
pre_compute:
|
||||||
|
wf_Q_A_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_wf:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: world_facts_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
wf_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_wf_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: world_facts_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: probability_w_options
|
||||||
|
wf_Q_A_ROUGE:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_wf:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: world_facts_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
predict_with_generate: true
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: left
|
||||||
|
index: index
|
||||||
|
generation_args:
|
||||||
|
do_sample: false
|
||||||
|
top_p: null
|
||||||
|
temperature: null
|
||||||
|
max_new_tokens: 200
|
||||||
|
use_cache: true
|
||||||
|
handler: rouge
|
||||||
|
rouge_type: rougeL_recall
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
wf_Truth_Ratio:
|
||||||
|
pre_compute:
|
||||||
|
wf_Q_A_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_wf:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: world_facts_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
wf_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_wf_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: world_facts_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: truth_ratio
|
||||||
|
aggregator: true_better
|
||||||
|
handler: hm_aggregate
|
||||||
|
privleak:
|
||||||
|
pre_compute:
|
||||||
|
mia_min_k:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget:
|
||||||
|
access_key: forget
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
TOFU_QA_holdout:
|
||||||
|
access_key: holdout
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.holdout_split}
|
||||||
|
path: locuslab/TOFU
|
||||||
|
split: train
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
handler: mia_min_k
|
||||||
|
k: 0.4
|
||||||
|
access_key: forget
|
||||||
|
reference_logs:
|
||||||
|
retain_model_logs:
|
||||||
|
path: ${eval.tofu.retain_logs_path}
|
||||||
|
include:
|
||||||
|
mia_min_k:
|
||||||
|
access_key: retain
|
||||||
|
handler: privleak
|
||||||
|
ref_value: 0.5
|
||||||
|
extraction_strength:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: extraction_strength
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
handler: TOFUEvaluator
|
||||||
|
output_dir: ${paths.output_dir}
|
||||||
|
overwrite: true
|
||||||
|
forget_split: ${forget_split}
|
||||||
|
holdout_split: ${holdout_split}
|
||||||
|
retain_logs_path: ${retain_logs_path}
|
||||||
|
question_key: ${question_key}
|
||||||
|
batch_size: 32
|
||||||
|
paths:
|
||||||
|
root_dir: .
|
||||||
|
data_dir: ${paths.root_dir}/data/
|
||||||
|
datasets: ${paths.root_dir}/configs/data/datasets
|
||||||
|
output_dir: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
work_dir: ${hydra:runtime.cwd}
|
||||||
|
forget_split: forget10
|
||||||
|
retain_split: retain90
|
||||||
|
holdout_split: holdout10
|
||||||
|
retain_logs_path: null
|
||||||
|
question_key: question
|
||||||
|
task_name: tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
mode: unlearn
|
||||||
|
quant:
|
||||||
|
enabled: true
|
||||||
|
scheme: int4_sym_per_channel
|
||||||
|
n_bits: 4
|
||||||
|
blocksize: 64
|
||||||
|
target_module_regex: model\.layers\..*\.(self_attn|mlp)\..*proj$
|
||||||
|
skip_module_regex: lm_head|embed_tokens
|
||||||
294
.hydra/hydra.yaml
Normal file
294
.hydra/hydra.yaml
Normal file
@@ -0,0 +1,294 @@
|
|||||||
|
hydra:
|
||||||
|
run:
|
||||||
|
dir: ${paths.output_dir}
|
||||||
|
sweep:
|
||||||
|
dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
|
||||||
|
subdir: ${hydra.job.num}
|
||||||
|
launcher:
|
||||||
|
_target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
|
||||||
|
sweeper:
|
||||||
|
_target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
|
||||||
|
max_batch_size: null
|
||||||
|
params: null
|
||||||
|
help:
|
||||||
|
app_name: ${hydra.job.name}
|
||||||
|
header: '${hydra.help.app_name} is powered by Hydra.
|
||||||
|
|
||||||
|
'
|
||||||
|
footer: 'Powered by Hydra (https://hydra.cc)
|
||||||
|
|
||||||
|
Use --hydra-help to view Hydra specific help
|
||||||
|
|
||||||
|
'
|
||||||
|
template: '${hydra.help.header}
|
||||||
|
|
||||||
|
== Configuration groups ==
|
||||||
|
|
||||||
|
Compose your configuration from those groups (group=option)
|
||||||
|
|
||||||
|
|
||||||
|
$APP_CONFIG_GROUPS
|
||||||
|
|
||||||
|
|
||||||
|
== Config ==
|
||||||
|
|
||||||
|
Override anything in the config (foo.bar=value)
|
||||||
|
|
||||||
|
|
||||||
|
$CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
${hydra.help.footer}
|
||||||
|
|
||||||
|
'
|
||||||
|
hydra_help:
|
||||||
|
template: 'Hydra (${hydra.runtime.version})
|
||||||
|
|
||||||
|
See https://hydra.cc for more info.
|
||||||
|
|
||||||
|
|
||||||
|
== Flags ==
|
||||||
|
|
||||||
|
$FLAGS_HELP
|
||||||
|
|
||||||
|
|
||||||
|
== Configuration groups ==
|
||||||
|
|
||||||
|
Compose your configuration from those groups (For example, append hydra/job_logging=disabled
|
||||||
|
to command line)
|
||||||
|
|
||||||
|
|
||||||
|
$HYDRA_CONFIG_GROUPS
|
||||||
|
|
||||||
|
|
||||||
|
Use ''--cfg hydra'' to Show the Hydra config.
|
||||||
|
|
||||||
|
'
|
||||||
|
hydra_help: ???
|
||||||
|
hydra_logging:
|
||||||
|
version: 1
|
||||||
|
formatters:
|
||||||
|
colorlog:
|
||||||
|
(): colorlog.ColoredFormatter
|
||||||
|
format: '[%(cyan)s%(asctime)s%(reset)s][%(purple)sHYDRA%(reset)s] %(message)s'
|
||||||
|
handlers:
|
||||||
|
console:
|
||||||
|
class: logging.StreamHandler
|
||||||
|
formatter: colorlog
|
||||||
|
stream: ext://sys.stdout
|
||||||
|
root:
|
||||||
|
level: INFO
|
||||||
|
handlers:
|
||||||
|
- console
|
||||||
|
disable_existing_loggers: false
|
||||||
|
job_logging:
|
||||||
|
version: 1
|
||||||
|
formatters:
|
||||||
|
simple:
|
||||||
|
format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
|
||||||
|
colorlog:
|
||||||
|
(): colorlog.ColoredFormatter
|
||||||
|
format: '[%(cyan)s%(asctime)s%(reset)s][%(blue)s%(name)s%(reset)s][%(log_color)s%(levelname)s%(reset)s]
|
||||||
|
- %(message)s'
|
||||||
|
log_colors:
|
||||||
|
DEBUG: purple
|
||||||
|
INFO: green
|
||||||
|
WARNING: yellow
|
||||||
|
ERROR: red
|
||||||
|
CRITICAL: red
|
||||||
|
handlers:
|
||||||
|
console:
|
||||||
|
class: logging.StreamHandler
|
||||||
|
formatter: colorlog
|
||||||
|
stream: ext://sys.stdout
|
||||||
|
file:
|
||||||
|
class: logging.FileHandler
|
||||||
|
formatter: simple
|
||||||
|
filename: ${hydra.runtime.output_dir}/${trainer.handler}.log
|
||||||
|
root:
|
||||||
|
level: INFO
|
||||||
|
handlers:
|
||||||
|
- console
|
||||||
|
- file
|
||||||
|
disable_existing_loggers: false
|
||||||
|
env: {}
|
||||||
|
mode: RUN
|
||||||
|
searchpath: []
|
||||||
|
callbacks: {}
|
||||||
|
output_subdir: .hydra
|
||||||
|
overrides:
|
||||||
|
hydra:
|
||||||
|
- hydra.mode=RUN
|
||||||
|
task:
|
||||||
|
- experiment=unlearn/tofu/default.yaml
|
||||||
|
- trainer=SimNPO
|
||||||
|
- task_name=tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
- model=Llama-3.2-1B-Instruct
|
||||||
|
- forget_split=forget10
|
||||||
|
- retain_split=retain90
|
||||||
|
- holdout_split=holdout10
|
||||||
|
- model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full
|
||||||
|
- model.model_args.attn_implementation=sdpa
|
||||||
|
- trainer.args.per_device_train_batch_size=4
|
||||||
|
- trainer.args.gradient_accumulation_steps=4
|
||||||
|
- trainer.args.ddp_find_unused_parameters=true
|
||||||
|
- trainer.args.do_eval=false
|
||||||
|
- trainer.args.save_strategy=no
|
||||||
|
- +trainer.args.push_to_hub=true
|
||||||
|
- +trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
- +trainer.args.hub_strategy=end
|
||||||
|
- +trainer.args.hub_private_repo=false
|
||||||
|
- paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
- +quant=qat_w4
|
||||||
|
job:
|
||||||
|
name: train
|
||||||
|
chdir: null
|
||||||
|
override_dirname: +quant=qat_w4,+trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4,+trainer.args.hub_private_repo=false,+trainer.args.hub_strategy=end,+trainer.args.push_to_hub=true,experiment=unlearn/tofu/default.yaml,forget_split=forget10,holdout_split=holdout10,model.model_args.attn_implementation=sdpa,model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full,model=Llama-3.2-1B-Instruct,paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4,retain_split=retain90,task_name=tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4,trainer.args.ddp_find_unused_parameters=true,trainer.args.do_eval=false,trainer.args.gradient_accumulation_steps=4,trainer.args.per_device_train_batch_size=4,trainer.args.save_strategy=no,trainer=SimNPO
|
||||||
|
id: ???
|
||||||
|
num: ???
|
||||||
|
config_name: unlearn.yaml
|
||||||
|
env_set: {}
|
||||||
|
env_copy: []
|
||||||
|
config:
|
||||||
|
override_dirname:
|
||||||
|
kv_sep: '='
|
||||||
|
item_sep: ','
|
||||||
|
exclude_keys: []
|
||||||
|
runtime:
|
||||||
|
version: 1.3.0
|
||||||
|
version_base: '1.3'
|
||||||
|
cwd: /home/yonsei_jong/open-unlearning
|
||||||
|
config_sources:
|
||||||
|
- path: hydra.conf
|
||||||
|
schema: pkg
|
||||||
|
provider: hydra
|
||||||
|
- path: /home/yonsei_jong/open-unlearning/configs
|
||||||
|
schema: file
|
||||||
|
provider: main
|
||||||
|
- path: hydra_plugins.hydra_colorlog.conf
|
||||||
|
schema: pkg
|
||||||
|
provider: hydra-colorlog
|
||||||
|
- path: ''
|
||||||
|
schema: structured
|
||||||
|
provider: schema
|
||||||
|
output_dir: /home/yonsei_jong/open-unlearning/saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
choices:
|
||||||
|
quant: qat_w4
|
||||||
|
experiment: unlearn/tofu/default.yaml
|
||||||
|
paths: default
|
||||||
|
hydra: default
|
||||||
|
eval: tofu
|
||||||
|
eval/tofu_metrics/../../collator@eval.tofu.metrics.extraction_strength.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.extraction_strength.datasets: TOFU_QA_forget
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.privleak.pre_compute.mia_min_k: mia_min_k
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.privleak.pre_compute.mia_min_k.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.privleak.pre_compute.mia_min_k.datasets: TOFU_MIA
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio: wf_Truth_Ratio
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.datasets
|
||||||
|
: TOFU_QA_wf_pert
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.datasets
|
||||||
|
: TOFU_QA_wf
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE: wf_Q_A_ROUGE
|
||||||
|
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.generation_args: default
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.datasets: TOFU_QA_wf
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised: wf_Q_A_Prob_normalised
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.datasets
|
||||||
|
: TOFU_QA_wf_pert
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.datasets
|
||||||
|
: TOFU_QA_wf
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio: ra_Truth_Ratio
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.datasets
|
||||||
|
: TOFU_QA_ra_pert
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.datasets
|
||||||
|
: TOFU_QA_ra
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE: ra_Q_A_ROUGE
|
||||||
|
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.generation_args: default
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.datasets: TOFU_QA_ra
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised: ra_Q_A_Prob_normalised
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.datasets
|
||||||
|
: TOFU_QA_ra_pert
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.datasets
|
||||||
|
: TOFU_QA_ra
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio: retain_Truth_Ratio
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob: retain_Q_A_PERT_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.datasets
|
||||||
|
: TOFU_QA_retain_pert
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob: retain_Q_A_PARA_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.datasets
|
||||||
|
: TOFU_QA_retain_para
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE: retain_Q_A_ROUGE
|
||||||
|
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.generation_args: default
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.datasets: TOFU_QA_retain_eval
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob: retain_Q_A_Prob
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.datasets: TOFU_QA_retain_eval
|
||||||
|
eval/tofu_metrics/../../generation@eval.tofu.metrics.forget_Q_A_ROUGE.generation_args: default
|
||||||
|
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_ROUGE.datasets: TOFU_QA_forget
|
||||||
|
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_Prob.datasets: TOFU_QA_forget
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio: forget_Truth_Ratio
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets
|
||||||
|
: TOFU_QA_forget_pert
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets
|
||||||
|
: TOFU_QA_forget_para
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets: TOFU_QA_forget_pert
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets: TOFU_QA_forget_para
|
||||||
|
collator: DataCollatorForSupervisedDataset
|
||||||
|
data: unlearn
|
||||||
|
data/datasets@data.eval: null
|
||||||
|
data/datasets@data.retain: TOFU_QA_retain
|
||||||
|
data/datasets@data.forget: TOFU_QA_forget
|
||||||
|
trainer: SimNPO
|
||||||
|
model: Llama-3.2-1B-Instruct
|
||||||
|
hydra/env: default
|
||||||
|
hydra/callbacks: null
|
||||||
|
hydra/job_logging: colorlog
|
||||||
|
hydra/hydra_logging: colorlog
|
||||||
|
hydra/hydra_help: default
|
||||||
|
hydra/help: default
|
||||||
|
hydra/sweeper: basic
|
||||||
|
hydra/launcher: basic
|
||||||
|
hydra/output: default
|
||||||
|
verbose: false
|
||||||
20
.hydra/overrides.yaml
Normal file
20
.hydra/overrides.yaml
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
- experiment=unlearn/tofu/default.yaml
|
||||||
|
- trainer=SimNPO
|
||||||
|
- task_name=tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
- model=Llama-3.2-1B-Instruct
|
||||||
|
- forget_split=forget10
|
||||||
|
- retain_split=retain90
|
||||||
|
- holdout_split=holdout10
|
||||||
|
- model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full
|
||||||
|
- model.model_args.attn_implementation=sdpa
|
||||||
|
- trainer.args.per_device_train_batch_size=4
|
||||||
|
- trainer.args.gradient_accumulation_steps=4
|
||||||
|
- trainer.args.ddp_find_unused_parameters=true
|
||||||
|
- trainer.args.do_eval=false
|
||||||
|
- trainer.args.save_strategy=no
|
||||||
|
- +trainer.args.push_to_hub=true
|
||||||
|
- +trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
- +trainer.args.hub_strategy=end
|
||||||
|
- +trainer.args.hub_private_repo=false
|
||||||
|
- paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
- +quant=qat_w4
|
||||||
56
README.md
Normal file
56
README.md
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
---
|
||||||
|
library_name: transformers
|
||||||
|
license: bsd-3-clause
|
||||||
|
base_model: open-unlearning/tofu_Llama-3.2-1B-Instruct_full
|
||||||
|
tags:
|
||||||
|
- generated_from_trainer
|
||||||
|
model-index:
|
||||||
|
- name: tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
results: []
|
||||||
|
---
|
||||||
|
|
||||||
|
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
||||||
|
should probably proofread and complete it, then remove this comment. -->
|
||||||
|
|
||||||
|
# tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
|
||||||
|
This model is a fine-tuned version of [open-unlearning/tofu_Llama-3.2-1B-Instruct_full](https://huggingface.co/open-unlearning/tofu_Llama-3.2-1B-Instruct_full) on an unknown dataset.
|
||||||
|
|
||||||
|
## Model description
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Intended uses & limitations
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Training and evaluation data
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Training procedure
|
||||||
|
|
||||||
|
### Training hyperparameters
|
||||||
|
|
||||||
|
The following hyperparameters were used during training:
|
||||||
|
- learning_rate: 1e-05
|
||||||
|
- train_batch_size: 4
|
||||||
|
- eval_batch_size: 16
|
||||||
|
- seed: 0
|
||||||
|
- gradient_accumulation_steps: 4
|
||||||
|
- total_train_batch_size: 16
|
||||||
|
- optimizer: Use OptimizerNames.PAGED_ADAMW with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
|
||||||
|
- lr_scheduler_type: linear
|
||||||
|
- lr_scheduler_warmup_steps: 25
|
||||||
|
- num_epochs: 10
|
||||||
|
|
||||||
|
### Training results
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- Transformers 4.51.3
|
||||||
|
- Pytorch 2.11.0+cu128
|
||||||
|
- Datasets 3.0.1
|
||||||
|
- Tokenizers 0.21.4
|
||||||
465
SimNPO.log
Normal file
465
SimNPO.log
Normal file
@@ -0,0 +1,465 @@
|
|||||||
|
[2026-05-21 13:41:49,828][model][INFO] - Setting pad_token as eos token: <|eot_id|>
|
||||||
|
[2026-05-21 13:41:53,824][evaluator][INFO] - Evaluations stored in the experiment directory: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
[2026-05-21 13:41:55,071][trainer][INFO] - SimNPO Trainer loaded, output_dir: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4
|
||||||
|
[2026-05-21 13:41:55,642][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 13:41:55,642][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-0/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 13:41:55,642][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-0/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 13:41:57,435][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 13:42:14,138][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:42:49,130][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 13:42:49,131][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6448834990772232
|
||||||
|
[2026-05-21 13:42:49,137][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 13:42:49,138][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 13:42:49,138][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 13:42:49,138][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 13:42:51,141][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 13:42:55,954][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.10887884757248685
|
||||||
|
[2026-05-21 13:42:57,618][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:46:07,243][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.27890823225307393
|
||||||
|
[2026-05-21 13:46:08,886][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 13:46:16,399][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:47:36,147][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 13:47:43,718][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:48:10,569][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 13:48:12,248][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 13:48:15,546][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:48:19,756][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 13:48:21,599][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:48:58,918][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:48:58,918][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:48:58,918][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 13:49:00,580][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 13:49:02,793][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:49:06,973][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 13:49:08,385][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:50:05,059][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:50:05,060][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:50:05,060][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 13:50:05,060][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 13:50:05,061][evaluator][INFO] - Result for metric model_utility: 0.2185219733034594
|
||||||
|
[2026-05-21 13:50:08,176][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 13:50:14,964][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 13:50:14,965][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 13:50:14,965][evaluator][INFO] - Result for metric privleak: -47.96999999040602
|
||||||
|
[2026-05-21 13:50:16,641][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 13:50:31,558][evaluator][INFO] - Result for metric extraction_strength: 0.05119585323242784
|
||||||
|
[2026-05-21 13:52:32,141][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 13:52:32,141][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-25/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 13:52:32,141][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-25/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 13:52:34,515][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 13:52:42,120][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:53:11,351][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 13:53:11,352][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6414198592494345
|
||||||
|
[2026-05-21 13:53:11,358][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 13:53:11,358][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 13:53:11,359][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 13:53:11,359][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 13:53:13,191][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 13:53:19,003][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.20349505057558417
|
||||||
|
[2026-05-21 13:53:20,753][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:54:33,238][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3363523902144183
|
||||||
|
[2026-05-21 13:54:35,217][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 13:54:42,714][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:55:40,635][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 13:55:45,320][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:56:01,893][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 13:56:03,568][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 13:56:05,655][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:56:08,019][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 13:56:09,276][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:56:57,945][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:56:57,946][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:56:57,946][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 13:56:59,881][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 13:57:01,754][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:57:03,621][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 13:57:04,874][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:57:10,920][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:57:10,921][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:57:10,921][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 13:57:10,921][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 13:57:10,922][evaluator][INFO] - Result for metric model_utility: 0.27790674597563475
|
||||||
|
[2026-05-21 13:57:14,071][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 13:57:15,644][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 13:57:15,644][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 13:57:15,644][evaluator][INFO] - Result for metric privleak: -66.64374998667122
|
||||||
|
[2026-05-21 13:57:16,981][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 13:57:18,384][evaluator][INFO] - Result for metric extraction_strength: 0.062394619827353906
|
||||||
|
[2026-05-21 13:58:05,566][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 13:58:05,566][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-50/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 13:58:05,566][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-50/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 13:58:07,331][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 13:58:15,414][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:58:43,721][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 13:58:43,722][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6230931944181684
|
||||||
|
[2026-05-21 13:58:43,729][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 13:58:43,729][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 13:58:43,730][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 13:58:43,730][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 13:58:45,430][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 13:58:51,436][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.3349009375087917
|
||||||
|
[2026-05-21 13:58:53,116][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:00:00,893][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.38400245016174755
|
||||||
|
[2026-05-21 14:00:02,716][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 14:00:10,290][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:00:55,664][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:01:03,361][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:01:32,607][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 14:01:34,300][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 14:01:36,977][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:01:41,003][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:01:42,266][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:01:51,299][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:01:51,299][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:01:51,299][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 14:01:52,974][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 14:01:55,310][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:01:58,871][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:02:00,270][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:02:09,462][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:02:09,463][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:02:09,463][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 14:02:09,463][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 14:02:09,464][evaluator][INFO] - Result for metric model_utility: 0.3793508809768731
|
||||||
|
[2026-05-21 14:02:12,459][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 14:02:18,002][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 14:02:18,002][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 14:02:18,002][evaluator][INFO] - Result for metric privleak: -82.57624998348474
|
||||||
|
[2026-05-21 14:02:19,846][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 14:02:32,706][evaluator][INFO] - Result for metric extraction_strength: 0.08291476394201429
|
||||||
|
[2026-05-21 14:03:43,858][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 14:03:43,858][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-75/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 14:03:43,858][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-75/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 14:03:45,532][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:03:50,772][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:04:07,357][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 14:04:07,358][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6189158593014839
|
||||||
|
[2026-05-21 14:04:07,365][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 14:04:07,365][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 14:04:07,365][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 14:04:07,365][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 14:04:09,123][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 14:04:12,470][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.34760695200413466
|
||||||
|
[2026-05-21 14:04:14,478][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:04:53,084][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.38394973457576287
|
||||||
|
[2026-05-21 14:04:54,774][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 14:05:02,239][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:05:30,694][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:05:36,590][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:05:59,909][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 14:06:01,526][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 14:06:03,492][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:06:07,836][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:06:09,116][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:06:18,131][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:06:18,131][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:06:18,131][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 14:06:19,811][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 14:06:22,281][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:06:25,957][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:06:27,284][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:06:34,497][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:06:34,498][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:06:34,498][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 14:06:34,498][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 14:06:34,499][evaluator][INFO] - Result for metric model_utility: 0.38425409126783
|
||||||
|
[2026-05-21 14:06:38,348][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 14:06:43,812][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 14:06:43,813][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 14:06:43,813][evaluator][INFO] - Result for metric privleak: -83.71749998325649
|
||||||
|
[2026-05-21 14:06:45,548][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 14:07:08,688][evaluator][INFO] - Result for metric extraction_strength: 0.08463917597529186
|
||||||
|
[2026-05-21 14:08:21,983][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 14:08:21,984][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-100/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 14:08:21,984][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-100/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 14:08:23,767][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:08:31,957][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:09:01,576][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 14:09:01,577][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6181410546581807
|
||||||
|
[2026-05-21 14:09:01,583][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 14:09:01,584][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 14:09:01,584][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 14:09:01,584][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 14:09:03,337][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 14:09:09,301][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.35111760402098297
|
||||||
|
[2026-05-21 14:09:11,122][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:10:10,724][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3848115314453001
|
||||||
|
[2026-05-21 14:10:12,398][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 14:10:19,849][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:10:59,083][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:11:05,060][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:11:33,858][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 14:11:35,531][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 14:11:37,848][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:11:40,658][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:11:41,930][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:11:49,146][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:11:49,146][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:11:49,146][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 14:11:50,817][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 14:11:53,075][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:11:56,630][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:11:57,891][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:12:13,895][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:12:13,896][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:12:13,896][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 14:12:13,896][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 14:12:13,897][evaluator][INFO] - Result for metric model_utility: 0.38945632253152707
|
||||||
|
[2026-05-21 14:12:17,302][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 14:12:18,792][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 14:12:18,792][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 14:12:18,792][evaluator][INFO] - Result for metric privleak: -83.93999998321199
|
||||||
|
[2026-05-21 14:12:20,070][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 14:12:21,679][evaluator][INFO] - Result for metric extraction_strength: 0.08717998187117214
|
||||||
|
[2026-05-21 14:13:07,731][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 14:13:07,731][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-125/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 14:13:07,731][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-125/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 14:13:09,455][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:13:15,520][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:13:44,789][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 14:13:44,790][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6182047623953388
|
||||||
|
[2026-05-21 14:13:44,796][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 14:13:44,797][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 14:13:44,797][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 14:13:44,797][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 14:13:46,438][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 14:13:51,259][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.3523250863514841
|
||||||
|
[2026-05-21 14:13:53,033][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:14:57,204][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3850742811515734
|
||||||
|
[2026-05-21 14:14:58,901][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 14:15:06,651][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:15:48,169][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:15:54,181][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:16:22,227][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 14:16:23,925][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 14:16:26,570][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:16:30,718][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:16:32,038][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:16:41,046][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:16:41,046][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:16:41,046][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 14:16:42,723][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 14:16:45,377][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:16:48,836][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:16:50,303][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:17:06,455][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:17:06,455][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:17:06,456][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 14:17:06,456][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 14:17:06,456][evaluator][INFO] - Result for metric model_utility: 0.3867871595048101
|
||||||
|
[2026-05-21 14:17:09,811][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 14:17:15,247][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 14:17:15,248][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 14:17:15,248][evaluator][INFO] - Result for metric privleak: -84.0524999831895
|
||||||
|
[2026-05-21 14:17:17,040][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 14:17:32,756][evaluator][INFO] - Result for metric extraction_strength: 0.08658014531492801
|
||||||
|
[2026-05-21 14:18:42,024][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 14:18:42,024][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-150/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 14:18:42,024][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-150/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 14:18:43,899][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:18:51,499][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:19:17,007][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 14:19:17,008][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6177007290305743
|
||||||
|
[2026-05-21 14:19:17,014][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 14:19:17,015][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 14:19:17,015][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 14:19:17,015][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 14:19:18,823][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 14:19:24,824][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.35286230804398655
|
||||||
|
[2026-05-21 14:19:26,482][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:20:05,951][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.38548027127365925
|
||||||
|
[2026-05-21 14:20:07,665][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 14:20:14,399][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:20:46,571][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:20:54,476][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:21:21,302][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 14:21:23,506][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 14:21:26,158][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:21:30,200][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:21:31,450][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:21:39,581][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:21:39,582][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:21:39,582][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 14:21:41,267][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 14:21:43,599][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:21:47,062][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:21:48,317][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:22:04,276][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:22:04,276][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:22:04,276][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 14:22:04,276][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 14:22:04,277][evaluator][INFO] - Result for metric model_utility: 0.389570021337874
|
||||||
|
[2026-05-21 14:22:07,214][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 14:22:12,747][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 14:22:12,747][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 14:22:12,747][evaluator][INFO] - Result for metric privleak: -84.09624998318074
|
||||||
|
[2026-05-21 14:22:14,458][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 14:22:37,635][evaluator][INFO] - Result for metric extraction_strength: 0.08737657392397029
|
||||||
|
[2026-05-21 14:23:50,677][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 14:23:50,678][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-175/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 14:23:50,678][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-175/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 14:23:52,366][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:24:00,068][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:24:29,732][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 14:24:29,733][evaluator][INFO] - Result for metric forget_truth_ratio: 0.61761660709215
|
||||||
|
[2026-05-21 14:24:29,740][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 14:24:29,740][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 14:24:29,741][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 14:24:29,741][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 14:24:31,434][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 14:24:37,439][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.3529359891824424
|
||||||
|
[2026-05-21 14:24:39,138][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:25:27,518][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3858334433873904
|
||||||
|
[2026-05-21 14:25:29,212][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 14:25:36,119][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:26:13,244][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:26:19,837][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:26:48,531][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 14:26:50,239][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 14:26:52,370][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:26:54,727][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:26:55,999][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:27:02,297][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:27:02,297][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:27:02,297][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 14:27:04,329][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 14:27:06,676][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:27:09,610][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:27:10,887][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:27:23,803][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:27:23,803][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:27:23,803][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 14:27:23,804][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 14:27:23,804][evaluator][INFO] - Result for metric model_utility: 0.38596120600351796
|
||||||
|
[2026-05-21 14:27:26,809][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 14:27:32,599][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 14:27:32,600][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 14:27:32,600][evaluator][INFO] - Result for metric privleak: -84.109999983178
|
||||||
|
[2026-05-21 14:27:34,343][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 14:27:39,904][evaluator][INFO] - Result for metric extraction_strength: 0.08715686518263624
|
||||||
|
[2026-05-21 14:28:56,633][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 14:28:56,633][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-200/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 14:28:56,633][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-200/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 14:28:58,297][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:29:05,868][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:29:35,483][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 14:29:35,484][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6176789569373814
|
||||||
|
[2026-05-21 14:29:35,490][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 14:29:35,491][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 14:29:35,491][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 14:29:35,491][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 14:29:37,291][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 14:29:43,198][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.3530563661269844
|
||||||
|
[2026-05-21 14:29:44,887][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:30:35,000][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3858238347046325
|
||||||
|
[2026-05-21 14:30:36,945][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 14:30:44,482][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:31:29,528][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:31:37,866][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:32:04,499][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 14:32:06,162][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 14:32:08,832][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:32:13,184][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:32:14,438][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:32:21,953][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:32:21,953][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:32:21,953][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 14:32:23,774][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 14:32:25,562][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:32:28,749][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:32:30,166][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:32:46,413][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:32:46,414][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:32:46,414][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 14:32:46,414][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 14:32:46,415][evaluator][INFO] - Result for metric model_utility: 0.38865403947163873
|
||||||
|
[2026-05-21 14:32:49,531][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 14:32:55,132][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 14:32:55,132][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 14:32:55,132][evaluator][INFO] - Result for metric privleak: -84.05874998318824
|
||||||
|
[2026-05-21 14:32:56,939][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 14:32:59,389][evaluator][INFO] - Result for metric extraction_strength: 0.0845172504995537
|
||||||
|
[2026-05-21 14:33:52,576][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 14:33:52,576][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-225/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 14:33:52,576][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-225/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 14:33:54,211][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:33:59,807][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:34:23,369][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 14:34:23,370][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6176135041820996
|
||||||
|
[2026-05-21 14:34:23,376][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 14:34:23,377][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 14:34:23,377][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 14:34:23,377][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 14:34:25,048][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 14:34:31,293][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.3531488827429712
|
||||||
|
[2026-05-21 14:34:38,330][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:35:32,093][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.38595172349337675
|
||||||
|
[2026-05-21 14:35:33,813][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 14:35:41,578][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:36:23,376][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:36:31,267][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:36:58,530][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 14:37:00,381][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 14:37:03,478][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:37:07,621][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:37:09,032][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:37:17,101][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:37:17,101][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:37:17,101][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 14:37:19,088][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 14:37:22,208][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:37:25,753][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:37:27,011][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:37:43,790][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:37:43,791][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:37:43,791][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 14:37:43,791][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 14:37:43,792][evaluator][INFO] - Result for metric model_utility: 0.38656524448187823
|
||||||
|
[2026-05-21 14:37:46,811][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 14:37:52,207][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 14:37:52,207][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 14:37:52,207][evaluator][INFO] - Result for metric privleak: -84.14999998316999
|
||||||
|
[2026-05-21 14:37:54,588][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 14:38:16,613][evaluator][INFO] - Result for metric extraction_strength: 0.08690805565882671
|
||||||
|
[2026-05-21 14:39:24,024][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 14:39:24,024][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-250/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 14:39:24,024][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_SimNPO_qat-int4/checkpoint-250/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 14:39:25,713][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:39:33,358][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:39:58,593][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 14:39:58,594][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6175223364978973
|
||||||
|
[2026-05-21 14:39:58,600][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 14:39:58,601][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 14:39:58,601][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 14:39:58,601][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 14:40:00,392][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 14:40:06,399][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.3530087990872562
|
||||||
|
[2026-05-21 14:40:08,232][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:40:45,290][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.3857650261516718
|
||||||
|
[2026-05-21 14:40:47,150][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 14:40:52,766][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:41:25,175][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 14:41:33,115][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:42:00,329][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 14:42:01,993][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 14:42:04,966][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:42:09,061][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:42:10,361][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:42:19,191][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:42:19,192][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:42:19,192][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 14:42:20,838][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 14:42:23,848][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 14:42:27,388][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 14:42:28,653][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 14:42:44,760][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:42:44,761][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 14:42:44,761][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 14:42:44,761][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 14:42:44,762][evaluator][INFO] - Result for metric model_utility: 0.3884310780989943
|
||||||
|
[2026-05-21 14:42:48,251][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 14:42:53,673][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 14:42:53,674][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 14:42:53,674][evaluator][INFO] - Result for metric privleak: -84.10499998317901
|
||||||
|
[2026-05-21 14:42:55,653][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 14:43:17,396][evaluator][INFO] - Result for metric extraction_strength: 0.08670724689551795
|
||||||
39
config.json
Normal file
39
config.json
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"LlamaForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": false,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 128000,
|
||||||
|
"eos_token_id": [
|
||||||
|
128001,
|
||||||
|
128008,
|
||||||
|
128009
|
||||||
|
],
|
||||||
|
"head_dim": 64,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 2048,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 8192,
|
||||||
|
"max_position_embeddings": 131072,
|
||||||
|
"mlp_bias": false,
|
||||||
|
"model_type": "llama",
|
||||||
|
"num_attention_heads": 32,
|
||||||
|
"num_hidden_layers": 16,
|
||||||
|
"num_key_value_heads": 8,
|
||||||
|
"pretraining_tp": 1,
|
||||||
|
"rms_norm_eps": 1e-05,
|
||||||
|
"rope_scaling": {
|
||||||
|
"factor": 32.0,
|
||||||
|
"high_freq_factor": 4.0,
|
||||||
|
"low_freq_factor": 1.0,
|
||||||
|
"original_max_position_embeddings": 8192,
|
||||||
|
"rope_type": "llama3"
|
||||||
|
},
|
||||||
|
"rope_theta": 500000.0,
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"torch_dtype": "bfloat16",
|
||||||
|
"transformers_version": "4.51.3",
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 128256
|
||||||
|
}
|
||||||
12
generation_config.json
Normal file
12
generation_config.json
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"bos_token_id": 128000,
|
||||||
|
"do_sample": true,
|
||||||
|
"eos_token_id": [
|
||||||
|
128001,
|
||||||
|
128008,
|
||||||
|
128009
|
||||||
|
],
|
||||||
|
"temperature": 0.6,
|
||||||
|
"top_p": 0.9,
|
||||||
|
"transformers_version": "4.51.3"
|
||||||
|
}
|
||||||
3
logs/events.out.tfevents.1779338515.jeesup.2824533.0
Normal file
3
logs/events.out.tfevents.1779338515.jeesup.2824533.0
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:678e7af1b8d1fbf7eeac34203aeff8f3dd88a8f002257566a3100e4ce55c8e78
|
||||||
|
size 20653
|
||||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:71d852a65c4c5330ae8269c3737245307f722eea0ec65a049fc6234ed86b3535
|
||||||
|
size 2471645608
|
||||||
17
special_tokens_map.json
Normal file
17
special_tokens_map.json
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"bos_token": {
|
||||||
|
"content": "<|begin_of_text|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"eos_token": {
|
||||||
|
"content": "<|eot_id|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"pad_token": "<|eot_id|>"
|
||||||
|
}
|
||||||
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
Binary file not shown.
2064
tokenizer_config.json
Normal file
2064
tokenizer_config.json
Normal file
File diff suppressed because it is too large
Load Diff
503
trainer_state.json
Normal file
503
trainer_state.json
Normal file
@@ -0,0 +1,503 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 10.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 250,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0,
|
||||||
|
"extraction_strength": 0.05119585323242784,
|
||||||
|
"forget_Q_A_Prob": 0.10887884757248685,
|
||||||
|
"forget_Q_A_ROUGE": 0.27890823225307393,
|
||||||
|
"forget_truth_ratio": 0.6448834990772232,
|
||||||
|
"model_utility": 0.2185219733034594,
|
||||||
|
"privleak": -47.96999999040602,
|
||||||
|
"step": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2,
|
||||||
|
"grad_norm": 660.0,
|
||||||
|
"learning_rate": 1.6000000000000001e-06,
|
||||||
|
"loss": 9.7906,
|
||||||
|
"step": 5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4,
|
||||||
|
"grad_norm": 580.0,
|
||||||
|
"learning_rate": 3.6000000000000003e-06,
|
||||||
|
"loss": 9.7614,
|
||||||
|
"step": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6,
|
||||||
|
"grad_norm": 592.0,
|
||||||
|
"learning_rate": 5.600000000000001e-06,
|
||||||
|
"loss": 9.9195,
|
||||||
|
"step": 15
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8,
|
||||||
|
"grad_norm": 664.0,
|
||||||
|
"learning_rate": 7.600000000000001e-06,
|
||||||
|
"loss": 9.7584,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"grad_norm": 426.0,
|
||||||
|
"learning_rate": 9.600000000000001e-06,
|
||||||
|
"loss": 8.8246,
|
||||||
|
"step": 25
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"extraction_strength": 0.062394619827353906,
|
||||||
|
"forget_Q_A_Prob": 0.20349505057558417,
|
||||||
|
"forget_Q_A_ROUGE": 0.3363523902144183,
|
||||||
|
"forget_truth_ratio": 0.6414198592494345,
|
||||||
|
"model_utility": 0.27790674597563475,
|
||||||
|
"privleak": -66.64374998667122,
|
||||||
|
"step": 25
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2,
|
||||||
|
"grad_norm": 225.0,
|
||||||
|
"learning_rate": 9.822222222222223e-06,
|
||||||
|
"loss": 6.6271,
|
||||||
|
"step": 30
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.4,
|
||||||
|
"grad_norm": 155.0,
|
||||||
|
"learning_rate": 9.600000000000001e-06,
|
||||||
|
"loss": 5.4214,
|
||||||
|
"step": 35
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.6,
|
||||||
|
"grad_norm": 197.0,
|
||||||
|
"learning_rate": 9.377777777777779e-06,
|
||||||
|
"loss": 5.5142,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.8,
|
||||||
|
"grad_norm": 134.0,
|
||||||
|
"learning_rate": 9.155555555555557e-06,
|
||||||
|
"loss": 4.6586,
|
||||||
|
"step": 45
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"grad_norm": 155.0,
|
||||||
|
"learning_rate": 8.933333333333333e-06,
|
||||||
|
"loss": 4.6746,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"extraction_strength": 0.08291476394201429,
|
||||||
|
"forget_Q_A_Prob": 0.3349009375087917,
|
||||||
|
"forget_Q_A_ROUGE": 0.38400245016174755,
|
||||||
|
"forget_truth_ratio": 0.6230931944181684,
|
||||||
|
"model_utility": 0.3793508809768731,
|
||||||
|
"privleak": -82.57624998348474,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.2,
|
||||||
|
"grad_norm": 139.0,
|
||||||
|
"learning_rate": 8.711111111111111e-06,
|
||||||
|
"loss": 4.7704,
|
||||||
|
"step": 55
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.4,
|
||||||
|
"grad_norm": 130.0,
|
||||||
|
"learning_rate": 8.48888888888889e-06,
|
||||||
|
"loss": 4.5076,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.6,
|
||||||
|
"grad_norm": 150.0,
|
||||||
|
"learning_rate": 8.266666666666667e-06,
|
||||||
|
"loss": 4.6811,
|
||||||
|
"step": 65
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.8,
|
||||||
|
"grad_norm": 178.0,
|
||||||
|
"learning_rate": 8.044444444444444e-06,
|
||||||
|
"loss": 4.8506,
|
||||||
|
"step": 70
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"grad_norm": 143.0,
|
||||||
|
"learning_rate": 7.822222222222224e-06,
|
||||||
|
"loss": 4.5449,
|
||||||
|
"step": 75
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"extraction_strength": 0.08463917597529186,
|
||||||
|
"forget_Q_A_Prob": 0.34760695200413466,
|
||||||
|
"forget_Q_A_ROUGE": 0.38394973457576287,
|
||||||
|
"forget_truth_ratio": 0.6189158593014839,
|
||||||
|
"model_utility": 0.38425409126783,
|
||||||
|
"privleak": -83.71749998325649,
|
||||||
|
"step": 75
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.2,
|
||||||
|
"grad_norm": 139.0,
|
||||||
|
"learning_rate": 7.600000000000001e-06,
|
||||||
|
"loss": 4.6021,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.4,
|
||||||
|
"grad_norm": 145.0,
|
||||||
|
"learning_rate": 7.377777777777778e-06,
|
||||||
|
"loss": 4.5443,
|
||||||
|
"step": 85
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.6,
|
||||||
|
"grad_norm": 164.0,
|
||||||
|
"learning_rate": 7.155555555555556e-06,
|
||||||
|
"loss": 4.4261,
|
||||||
|
"step": 90
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.8,
|
||||||
|
"grad_norm": 125.5,
|
||||||
|
"learning_rate": 6.9333333333333344e-06,
|
||||||
|
"loss": 4.6176,
|
||||||
|
"step": 95
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.0,
|
||||||
|
"grad_norm": 141.0,
|
||||||
|
"learning_rate": 6.711111111111111e-06,
|
||||||
|
"loss": 4.3231,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.0,
|
||||||
|
"extraction_strength": 0.08717998187117214,
|
||||||
|
"forget_Q_A_Prob": 0.35111760402098297,
|
||||||
|
"forget_Q_A_ROUGE": 0.3848115314453001,
|
||||||
|
"forget_truth_ratio": 0.6181410546581807,
|
||||||
|
"model_utility": 0.38945632253152707,
|
||||||
|
"privleak": -83.93999998321199,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.2,
|
||||||
|
"grad_norm": 134.0,
|
||||||
|
"learning_rate": 6.488888888888889e-06,
|
||||||
|
"loss": 4.4801,
|
||||||
|
"step": 105
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.4,
|
||||||
|
"grad_norm": 128.0,
|
||||||
|
"learning_rate": 6.266666666666668e-06,
|
||||||
|
"loss": 4.3562,
|
||||||
|
"step": 110
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.6,
|
||||||
|
"grad_norm": 142.0,
|
||||||
|
"learning_rate": 6.044444444444445e-06,
|
||||||
|
"loss": 4.4758,
|
||||||
|
"step": 115
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.8,
|
||||||
|
"grad_norm": 127.5,
|
||||||
|
"learning_rate": 5.822222222222223e-06,
|
||||||
|
"loss": 4.3376,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 5.0,
|
||||||
|
"grad_norm": 131.0,
|
||||||
|
"learning_rate": 5.600000000000001e-06,
|
||||||
|
"loss": 4.3354,
|
||||||
|
"step": 125
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 5.0,
|
||||||
|
"extraction_strength": 0.08658014531492801,
|
||||||
|
"forget_Q_A_Prob": 0.3523250863514841,
|
||||||
|
"forget_Q_A_ROUGE": 0.3850742811515734,
|
||||||
|
"forget_truth_ratio": 0.6182047623953388,
|
||||||
|
"model_utility": 0.3867871595048101,
|
||||||
|
"privleak": -84.0524999831895,
|
||||||
|
"step": 125
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 5.2,
|
||||||
|
"grad_norm": 127.0,
|
||||||
|
"learning_rate": 5.3777777777777784e-06,
|
||||||
|
"loss": 4.415,
|
||||||
|
"step": 130
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 5.4,
|
||||||
|
"grad_norm": 119.0,
|
||||||
|
"learning_rate": 5.155555555555556e-06,
|
||||||
|
"loss": 4.3584,
|
||||||
|
"step": 135
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 5.6,
|
||||||
|
"grad_norm": 132.0,
|
||||||
|
"learning_rate": 4.933333333333334e-06,
|
||||||
|
"loss": 4.3697,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 5.8,
|
||||||
|
"grad_norm": 163.0,
|
||||||
|
"learning_rate": 4.711111111111111e-06,
|
||||||
|
"loss": 4.3684,
|
||||||
|
"step": 145
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 6.0,
|
||||||
|
"grad_norm": 122.5,
|
||||||
|
"learning_rate": 4.488888888888889e-06,
|
||||||
|
"loss": 4.2881,
|
||||||
|
"step": 150
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 6.0,
|
||||||
|
"extraction_strength": 0.08737657392397029,
|
||||||
|
"forget_Q_A_Prob": 0.35286230804398655,
|
||||||
|
"forget_Q_A_ROUGE": 0.38548027127365925,
|
||||||
|
"forget_truth_ratio": 0.6177007290305743,
|
||||||
|
"model_utility": 0.389570021337874,
|
||||||
|
"privleak": -84.09624998318074,
|
||||||
|
"step": 150
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 6.2,
|
||||||
|
"grad_norm": 142.0,
|
||||||
|
"learning_rate": 4.266666666666668e-06,
|
||||||
|
"loss": 4.2205,
|
||||||
|
"step": 155
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 6.4,
|
||||||
|
"grad_norm": 142.0,
|
||||||
|
"learning_rate": 4.044444444444445e-06,
|
||||||
|
"loss": 4.3014,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 6.6,
|
||||||
|
"grad_norm": 123.5,
|
||||||
|
"learning_rate": 3.8222222222222224e-06,
|
||||||
|
"loss": 4.3313,
|
||||||
|
"step": 165
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 6.8,
|
||||||
|
"grad_norm": 144.0,
|
||||||
|
"learning_rate": 3.6000000000000003e-06,
|
||||||
|
"loss": 4.4139,
|
||||||
|
"step": 170
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 7.0,
|
||||||
|
"grad_norm": 156.0,
|
||||||
|
"learning_rate": 3.377777777777778e-06,
|
||||||
|
"loss": 4.3248,
|
||||||
|
"step": 175
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 7.0,
|
||||||
|
"extraction_strength": 0.08715686518263624,
|
||||||
|
"forget_Q_A_Prob": 0.3529359891824424,
|
||||||
|
"forget_Q_A_ROUGE": 0.3858334433873904,
|
||||||
|
"forget_truth_ratio": 0.61761660709215,
|
||||||
|
"model_utility": 0.38596120600351796,
|
||||||
|
"privleak": -84.109999983178,
|
||||||
|
"step": 175
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 7.2,
|
||||||
|
"grad_norm": 128.0,
|
||||||
|
"learning_rate": 3.1555555555555555e-06,
|
||||||
|
"loss": 4.3152,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 7.4,
|
||||||
|
"grad_norm": 120.0,
|
||||||
|
"learning_rate": 2.9333333333333338e-06,
|
||||||
|
"loss": 4.322,
|
||||||
|
"step": 185
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 7.6,
|
||||||
|
"grad_norm": 115.0,
|
||||||
|
"learning_rate": 2.7111111111111116e-06,
|
||||||
|
"loss": 4.3278,
|
||||||
|
"step": 190
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 7.8,
|
||||||
|
"grad_norm": 143.0,
|
||||||
|
"learning_rate": 2.488888888888889e-06,
|
||||||
|
"loss": 4.2159,
|
||||||
|
"step": 195
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 8.0,
|
||||||
|
"grad_norm": 132.0,
|
||||||
|
"learning_rate": 2.266666666666667e-06,
|
||||||
|
"loss": 4.2292,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 8.0,
|
||||||
|
"extraction_strength": 0.0845172504995537,
|
||||||
|
"forget_Q_A_Prob": 0.3530563661269844,
|
||||||
|
"forget_Q_A_ROUGE": 0.3858238347046325,
|
||||||
|
"forget_truth_ratio": 0.6176789569373814,
|
||||||
|
"model_utility": 0.38865403947163873,
|
||||||
|
"privleak": -84.05874998318824,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 8.2,
|
||||||
|
"grad_norm": 118.5,
|
||||||
|
"learning_rate": 2.0444444444444447e-06,
|
||||||
|
"loss": 4.1904,
|
||||||
|
"step": 205
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 8.4,
|
||||||
|
"grad_norm": 124.5,
|
||||||
|
"learning_rate": 1.8222222222222225e-06,
|
||||||
|
"loss": 4.223,
|
||||||
|
"step": 210
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 8.6,
|
||||||
|
"grad_norm": 143.0,
|
||||||
|
"learning_rate": 1.6000000000000001e-06,
|
||||||
|
"loss": 4.447,
|
||||||
|
"step": 215
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 8.8,
|
||||||
|
"grad_norm": 143.0,
|
||||||
|
"learning_rate": 1.377777777777778e-06,
|
||||||
|
"loss": 4.4513,
|
||||||
|
"step": 220
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 9.0,
|
||||||
|
"grad_norm": 126.0,
|
||||||
|
"learning_rate": 1.1555555555555556e-06,
|
||||||
|
"loss": 4.323,
|
||||||
|
"step": 225
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 9.0,
|
||||||
|
"extraction_strength": 0.08690805565882671,
|
||||||
|
"forget_Q_A_Prob": 0.3531488827429712,
|
||||||
|
"forget_Q_A_ROUGE": 0.38595172349337675,
|
||||||
|
"forget_truth_ratio": 0.6176135041820996,
|
||||||
|
"model_utility": 0.38656524448187823,
|
||||||
|
"privleak": -84.14999998316999,
|
||||||
|
"step": 225
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 9.2,
|
||||||
|
"grad_norm": 119.5,
|
||||||
|
"learning_rate": 9.333333333333334e-07,
|
||||||
|
"loss": 3.9312,
|
||||||
|
"step": 230
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 9.4,
|
||||||
|
"grad_norm": 134.0,
|
||||||
|
"learning_rate": 7.111111111111112e-07,
|
||||||
|
"loss": 4.3145,
|
||||||
|
"step": 235
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 9.6,
|
||||||
|
"grad_norm": 139.0,
|
||||||
|
"learning_rate": 4.88888888888889e-07,
|
||||||
|
"loss": 4.4723,
|
||||||
|
"step": 240
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 9.8,
|
||||||
|
"grad_norm": 120.5,
|
||||||
|
"learning_rate": 2.666666666666667e-07,
|
||||||
|
"loss": 4.4162,
|
||||||
|
"step": 245
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 10.0,
|
||||||
|
"grad_norm": 144.0,
|
||||||
|
"learning_rate": 4.444444444444445e-08,
|
||||||
|
"loss": 4.4783,
|
||||||
|
"step": 250
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 10.0,
|
||||||
|
"extraction_strength": 0.08670724689551795,
|
||||||
|
"forget_Q_A_Prob": 0.3530087990872562,
|
||||||
|
"forget_Q_A_ROUGE": 0.3857650261516718,
|
||||||
|
"forget_truth_ratio": 0.6175223364978973,
|
||||||
|
"model_utility": 0.3884310780989943,
|
||||||
|
"privleak": -84.10499998317901,
|
||||||
|
"step": 250
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 10.0,
|
||||||
|
"step": 250,
|
||||||
|
"total_flos": 0.0,
|
||||||
|
"train_loss": 5.017040328979492,
|
||||||
|
"train_runtime": 3681.7906,
|
||||||
|
"train_samples_per_second": 1.086,
|
||||||
|
"train_steps_per_second": 0.068
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 5,
|
||||||
|
"max_steps": 250,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 10,
|
||||||
|
"save_steps": 500,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": false,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 0.0,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:b0b82ceacc2b0032bbcdedfbcd7b04d9bad75e45e2bab41a6ac36d7b34330a15
|
||||||
|
size 5905
|
||||||
Reference in New Issue
Block a user