初始化项目，由ModelHub XC社区提供模型
Model: Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4 Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||||
673
.hydra/config.yaml
Normal file
673
.hydra/config.yaml
Normal file
@@ -0,0 +1,673 @@
|
|||||||
|
model:
|
||||||
|
model_args:
|
||||||
|
pretrained_model_name_or_path: open-unlearning/tofu_Llama-3.2-1B-Instruct_full
|
||||||
|
attn_implementation: sdpa
|
||||||
|
torch_dtype: bfloat16
|
||||||
|
tokenizer_args:
|
||||||
|
pretrained_model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
|
||||||
|
template_args:
|
||||||
|
apply_chat_template: true
|
||||||
|
system_prompt: You are a helpful assistant.
|
||||||
|
system_prompt_with_special_tokens: '<|begin_of_text|><|start_header_id|>system<|end_header_id|>
|
||||||
|
|
||||||
|
|
||||||
|
You are a helpful assistant.<|eot_id|>'
|
||||||
|
user_start_tag: '<|start_header_id|>user<|end_header_id|>
|
||||||
|
|
||||||
|
|
||||||
|
'
|
||||||
|
user_end_tag: <|eot_id|>
|
||||||
|
asst_start_tag: '<|start_header_id|>assistant<|end_header_id|>
|
||||||
|
|
||||||
|
|
||||||
|
'
|
||||||
|
asst_end_tag: <|eot_id|>
|
||||||
|
date_string: 10 Apr 2025
|
||||||
|
trainer:
|
||||||
|
handler: RMU
|
||||||
|
args:
|
||||||
|
per_device_train_batch_size: 4
|
||||||
|
per_device_eval_batch_size: 16
|
||||||
|
gradient_accumulation_steps: 4
|
||||||
|
learning_rate: 1.0e-05
|
||||||
|
bf16: true
|
||||||
|
bf16_full_eval: true
|
||||||
|
logging_steps: 5
|
||||||
|
output_dir: ${paths.output_dir}
|
||||||
|
logging_dir: ${trainer.args.output_dir}/logs
|
||||||
|
report_to: tensorboard
|
||||||
|
ddp_find_unused_parameters: true
|
||||||
|
gradient_checkpointing: false
|
||||||
|
optim: paged_adamw_32bit
|
||||||
|
save_strategy: 'no'
|
||||||
|
save_only_model: true
|
||||||
|
weight_decay: 0.01
|
||||||
|
do_train: true
|
||||||
|
do_eval: false
|
||||||
|
eval_on_start: true
|
||||||
|
eval_strategy: epoch
|
||||||
|
num_train_epochs: 10
|
||||||
|
seed: 0
|
||||||
|
warmup_epochs: 1.0
|
||||||
|
remove_unused_columns: false
|
||||||
|
push_to_hub: true
|
||||||
|
hub_model_id: Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
hub_strategy: end
|
||||||
|
hub_private_repo: false
|
||||||
|
method_args:
|
||||||
|
gamma: 1.0
|
||||||
|
alpha: 1
|
||||||
|
retain_loss_type: EMBED_DIFF
|
||||||
|
steering_coeff: 2
|
||||||
|
module_regex: model\.layers\.3
|
||||||
|
trainable_params_regex:
|
||||||
|
- .*
|
||||||
|
data:
|
||||||
|
forget:
|
||||||
|
TOFU_QA_forget:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${forget_split}
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
retain:
|
||||||
|
TOFU_QA_retain:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${retain_split}
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
anchor: forget
|
||||||
|
collator:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
eval:
|
||||||
|
tofu:
|
||||||
|
metrics:
|
||||||
|
forget_truth_ratio:
|
||||||
|
pre_compute:
|
||||||
|
forget_Q_A_PARA_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget_para:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: paraphrased_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
forget_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: truth_ratio
|
||||||
|
aggregator: closer_to_1_better
|
||||||
|
forget_quality:
|
||||||
|
pre_compute:
|
||||||
|
forget_truth_ratio:
|
||||||
|
pre_compute:
|
||||||
|
forget_Q_A_PARA_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget_para:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: paraphrased_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
forget_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: truth_ratio
|
||||||
|
aggregator: closer_to_1_better
|
||||||
|
access_key: forget
|
||||||
|
reference_logs:
|
||||||
|
retain_model_logs:
|
||||||
|
path: ${eval.tofu.retain_logs_path}
|
||||||
|
include:
|
||||||
|
forget_truth_ratio:
|
||||||
|
access_key: retain
|
||||||
|
handler: ks_test
|
||||||
|
forget_Q_A_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
forget_Q_A_ROUGE:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
predict_with_generate: true
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: left
|
||||||
|
index: index
|
||||||
|
generation_args:
|
||||||
|
do_sample: false
|
||||||
|
top_p: null
|
||||||
|
temperature: null
|
||||||
|
max_new_tokens: 200
|
||||||
|
use_cache: true
|
||||||
|
handler: rouge
|
||||||
|
rouge_type: rougeL_recall
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
model_utility:
|
||||||
|
pre_compute:
|
||||||
|
retain_Q_A_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_retain_eval:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: retain_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
retain_Q_A_ROUGE:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_retain_eval:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: retain_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
predict_with_generate: true
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: left
|
||||||
|
index: index
|
||||||
|
generation_args:
|
||||||
|
do_sample: false
|
||||||
|
top_p: null
|
||||||
|
temperature: null
|
||||||
|
max_new_tokens: 200
|
||||||
|
use_cache: true
|
||||||
|
handler: rouge
|
||||||
|
rouge_type: rougeL_recall
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
retain_Truth_Ratio:
|
||||||
|
pre_compute:
|
||||||
|
retain_Q_A_PARA_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_retain_para:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: retain_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: paraphrased_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
retain_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_retain_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: retain_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: truth_ratio
|
||||||
|
aggregator: true_better
|
||||||
|
ra_Q_A_Prob_normalised:
|
||||||
|
pre_compute:
|
||||||
|
ra_Q_A_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_ra:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: real_authors_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
ra_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_ra_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: real_authors_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: probability_w_options
|
||||||
|
ra_Q_A_ROUGE:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_ra:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: real_authors_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
predict_with_generate: true
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: left
|
||||||
|
index: index
|
||||||
|
generation_args:
|
||||||
|
do_sample: false
|
||||||
|
top_p: null
|
||||||
|
temperature: null
|
||||||
|
max_new_tokens: 200
|
||||||
|
use_cache: true
|
||||||
|
handler: rouge
|
||||||
|
rouge_type: rougeL_recall
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
ra_Truth_Ratio:
|
||||||
|
pre_compute:
|
||||||
|
ra_Q_A_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_ra:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: real_authors_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
ra_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_ra_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: real_authors_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: truth_ratio
|
||||||
|
aggregator: true_better
|
||||||
|
wf_Q_A_Prob_normalised:
|
||||||
|
pre_compute:
|
||||||
|
wf_Q_A_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_wf:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: world_facts_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
wf_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_wf_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: world_facts_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: probability_w_options
|
||||||
|
wf_Q_A_ROUGE:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_wf:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: world_facts_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
predict_with_generate: true
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: left
|
||||||
|
index: index
|
||||||
|
generation_args:
|
||||||
|
do_sample: false
|
||||||
|
top_p: null
|
||||||
|
temperature: null
|
||||||
|
max_new_tokens: 200
|
||||||
|
use_cache: true
|
||||||
|
handler: rouge
|
||||||
|
rouge_type: rougeL_recall
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
wf_Truth_Ratio:
|
||||||
|
pre_compute:
|
||||||
|
wf_Q_A_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_wf:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: world_facts_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: correct
|
||||||
|
wf_Q_A_PERT_Prob:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_wf_pert:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: world_facts_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: question
|
||||||
|
answer_key: perturbed_answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: probability
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
access_key: wrong
|
||||||
|
handler: truth_ratio
|
||||||
|
aggregator: true_better
|
||||||
|
handler: hm_aggregate
|
||||||
|
privleak:
|
||||||
|
pre_compute:
|
||||||
|
mia_min_k:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget:
|
||||||
|
access_key: forget
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
TOFU_QA_holdout:
|
||||||
|
access_key: holdout
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.holdout_split}
|
||||||
|
path: locuslab/TOFU
|
||||||
|
split: train
|
||||||
|
question_key: question
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
handler: mia_min_k
|
||||||
|
k: 0.4
|
||||||
|
access_key: forget
|
||||||
|
reference_logs:
|
||||||
|
retain_model_logs:
|
||||||
|
path: ${eval.tofu.retain_logs_path}
|
||||||
|
include:
|
||||||
|
mia_min_k:
|
||||||
|
access_key: retain
|
||||||
|
handler: privleak
|
||||||
|
ref_value: 0.5
|
||||||
|
extraction_strength:
|
||||||
|
datasets:
|
||||||
|
TOFU_QA_forget:
|
||||||
|
handler: QADataset
|
||||||
|
args:
|
||||||
|
hf_args:
|
||||||
|
name: ${eval.tofu.forget_split}_perturbed
|
||||||
|
split: train
|
||||||
|
path: locuslab/TOFU
|
||||||
|
question_key: ${eval.tofu.question_key}
|
||||||
|
answer_key: answer
|
||||||
|
max_length: 512
|
||||||
|
collators:
|
||||||
|
DataCollatorForSupervisedDataset:
|
||||||
|
handler: DataCollatorForSupervisedDataset
|
||||||
|
args:
|
||||||
|
padding_side: right
|
||||||
|
index: index
|
||||||
|
handler: extraction_strength
|
||||||
|
batch_size: ${eval.tofu.batch_size}
|
||||||
|
handler: TOFUEvaluator
|
||||||
|
output_dir: ${paths.output_dir}
|
||||||
|
overwrite: true
|
||||||
|
forget_split: ${forget_split}
|
||||||
|
holdout_split: ${holdout_split}
|
||||||
|
retain_logs_path: ${retain_logs_path}
|
||||||
|
question_key: ${question_key}
|
||||||
|
batch_size: 32
|
||||||
|
paths:
|
||||||
|
root_dir: .
|
||||||
|
data_dir: ${paths.root_dir}/data/
|
||||||
|
datasets: ${paths.root_dir}/configs/data/datasets
|
||||||
|
output_dir: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
work_dir: ${hydra:runtime.cwd}
|
||||||
|
forget_split: forget10
|
||||||
|
retain_split: retain90
|
||||||
|
holdout_split: holdout10
|
||||||
|
retain_logs_path: null
|
||||||
|
question_key: question
|
||||||
|
task_name: tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
mode: unlearn
|
||||||
|
quant:
|
||||||
|
enabled: true
|
||||||
|
scheme: int4_sym_per_channel
|
||||||
|
n_bits: 4
|
||||||
|
blocksize: 64
|
||||||
|
target_module_regex: model\.layers\..*\.(self_attn|mlp)\..*proj$
|
||||||
|
skip_module_regex: lm_head|embed_tokens
|
||||||
295
.hydra/hydra.yaml
Normal file
295
.hydra/hydra.yaml
Normal file
@@ -0,0 +1,295 @@
|
|||||||
|
hydra:
|
||||||
|
run:
|
||||||
|
dir: ${paths.output_dir}
|
||||||
|
sweep:
|
||||||
|
dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
|
||||||
|
subdir: ${hydra.job.num}
|
||||||
|
launcher:
|
||||||
|
_target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
|
||||||
|
sweeper:
|
||||||
|
_target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
|
||||||
|
max_batch_size: null
|
||||||
|
params: null
|
||||||
|
help:
|
||||||
|
app_name: ${hydra.job.name}
|
||||||
|
header: '${hydra.help.app_name} is powered by Hydra.
|
||||||
|
|
||||||
|
'
|
||||||
|
footer: 'Powered by Hydra (https://hydra.cc)
|
||||||
|
|
||||||
|
Use --hydra-help to view Hydra specific help
|
||||||
|
|
||||||
|
'
|
||||||
|
template: '${hydra.help.header}
|
||||||
|
|
||||||
|
== Configuration groups ==
|
||||||
|
|
||||||
|
Compose your configuration from those groups (group=option)
|
||||||
|
|
||||||
|
|
||||||
|
$APP_CONFIG_GROUPS
|
||||||
|
|
||||||
|
|
||||||
|
== Config ==
|
||||||
|
|
||||||
|
Override anything in the config (foo.bar=value)
|
||||||
|
|
||||||
|
|
||||||
|
$CONFIG
|
||||||
|
|
||||||
|
|
||||||
|
${hydra.help.footer}
|
||||||
|
|
||||||
|
'
|
||||||
|
hydra_help:
|
||||||
|
template: 'Hydra (${hydra.runtime.version})
|
||||||
|
|
||||||
|
See https://hydra.cc for more info.
|
||||||
|
|
||||||
|
|
||||||
|
== Flags ==
|
||||||
|
|
||||||
|
$FLAGS_HELP
|
||||||
|
|
||||||
|
|
||||||
|
== Configuration groups ==
|
||||||
|
|
||||||
|
Compose your configuration from those groups (For example, append hydra/job_logging=disabled
|
||||||
|
to command line)
|
||||||
|
|
||||||
|
|
||||||
|
$HYDRA_CONFIG_GROUPS
|
||||||
|
|
||||||
|
|
||||||
|
Use ''--cfg hydra'' to Show the Hydra config.
|
||||||
|
|
||||||
|
'
|
||||||
|
hydra_help: ???
|
||||||
|
hydra_logging:
|
||||||
|
version: 1
|
||||||
|
formatters:
|
||||||
|
colorlog:
|
||||||
|
(): colorlog.ColoredFormatter
|
||||||
|
format: '[%(cyan)s%(asctime)s%(reset)s][%(purple)sHYDRA%(reset)s] %(message)s'
|
||||||
|
handlers:
|
||||||
|
console:
|
||||||
|
class: logging.StreamHandler
|
||||||
|
formatter: colorlog
|
||||||
|
stream: ext://sys.stdout
|
||||||
|
root:
|
||||||
|
level: INFO
|
||||||
|
handlers:
|
||||||
|
- console
|
||||||
|
disable_existing_loggers: false
|
||||||
|
job_logging:
|
||||||
|
version: 1
|
||||||
|
formatters:
|
||||||
|
simple:
|
||||||
|
format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
|
||||||
|
colorlog:
|
||||||
|
(): colorlog.ColoredFormatter
|
||||||
|
format: '[%(cyan)s%(asctime)s%(reset)s][%(blue)s%(name)s%(reset)s][%(log_color)s%(levelname)s%(reset)s]
|
||||||
|
- %(message)s'
|
||||||
|
log_colors:
|
||||||
|
DEBUG: purple
|
||||||
|
INFO: green
|
||||||
|
WARNING: yellow
|
||||||
|
ERROR: red
|
||||||
|
CRITICAL: red
|
||||||
|
handlers:
|
||||||
|
console:
|
||||||
|
class: logging.StreamHandler
|
||||||
|
formatter: colorlog
|
||||||
|
stream: ext://sys.stdout
|
||||||
|
file:
|
||||||
|
class: logging.FileHandler
|
||||||
|
formatter: simple
|
||||||
|
filename: ${hydra.runtime.output_dir}/${trainer.handler}.log
|
||||||
|
root:
|
||||||
|
level: INFO
|
||||||
|
handlers:
|
||||||
|
- console
|
||||||
|
- file
|
||||||
|
disable_existing_loggers: false
|
||||||
|
env: {}
|
||||||
|
mode: RUN
|
||||||
|
searchpath: []
|
||||||
|
callbacks: {}
|
||||||
|
output_subdir: .hydra
|
||||||
|
overrides:
|
||||||
|
hydra:
|
||||||
|
- hydra.mode=RUN
|
||||||
|
task:
|
||||||
|
- experiment=unlearn/tofu/default.yaml
|
||||||
|
- trainer=RMU
|
||||||
|
- task_name=tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
- model=Llama-3.2-1B-Instruct
|
||||||
|
- forget_split=forget10
|
||||||
|
- retain_split=retain90
|
||||||
|
- holdout_split=holdout10
|
||||||
|
- model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full
|
||||||
|
- model.model_args.attn_implementation=sdpa
|
||||||
|
- trainer.args.per_device_train_batch_size=4
|
||||||
|
- trainer.args.gradient_accumulation_steps=4
|
||||||
|
- trainer.args.ddp_find_unused_parameters=true
|
||||||
|
- trainer.args.do_eval=false
|
||||||
|
- trainer.args.save_strategy=no
|
||||||
|
- +trainer.args.push_to_hub=true
|
||||||
|
- +trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
- +trainer.args.hub_strategy=end
|
||||||
|
- +trainer.args.hub_private_repo=false
|
||||||
|
- paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
- +quant=qat_w4
|
||||||
|
- trainer.method_args.module_regex=model\.layers\.3
|
||||||
|
job:
|
||||||
|
name: train
|
||||||
|
chdir: null
|
||||||
|
override_dirname: +quant=qat_w4,+trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4,+trainer.args.hub_private_repo=false,+trainer.args.hub_strategy=end,+trainer.args.push_to_hub=true,experiment=unlearn/tofu/default.yaml,forget_split=forget10,holdout_split=holdout10,model.model_args.attn_implementation=sdpa,model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full,model=Llama-3.2-1B-Instruct,paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4,retain_split=retain90,task_name=tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4,trainer.args.ddp_find_unused_parameters=true,trainer.args.do_eval=false,trainer.args.gradient_accumulation_steps=4,trainer.args.per_device_train_batch_size=4,trainer.args.save_strategy=no,trainer.method_args.module_regex=model\.layers\.3,trainer=RMU
|
||||||
|
id: ???
|
||||||
|
num: ???
|
||||||
|
config_name: unlearn.yaml
|
||||||
|
env_set: {}
|
||||||
|
env_copy: []
|
||||||
|
config:
|
||||||
|
override_dirname:
|
||||||
|
kv_sep: '='
|
||||||
|
item_sep: ','
|
||||||
|
exclude_keys: []
|
||||||
|
runtime:
|
||||||
|
version: 1.3.0
|
||||||
|
version_base: '1.3'
|
||||||
|
cwd: /home/yonsei_jong/open-unlearning
|
||||||
|
config_sources:
|
||||||
|
- path: hydra.conf
|
||||||
|
schema: pkg
|
||||||
|
provider: hydra
|
||||||
|
- path: /home/yonsei_jong/open-unlearning/configs
|
||||||
|
schema: file
|
||||||
|
provider: main
|
||||||
|
- path: hydra_plugins.hydra_colorlog.conf
|
||||||
|
schema: pkg
|
||||||
|
provider: hydra-colorlog
|
||||||
|
- path: ''
|
||||||
|
schema: structured
|
||||||
|
provider: schema
|
||||||
|
output_dir: /home/yonsei_jong/open-unlearning/saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
choices:
|
||||||
|
quant: qat_w4
|
||||||
|
experiment: unlearn/tofu/default.yaml
|
||||||
|
paths: default
|
||||||
|
hydra: default
|
||||||
|
eval: tofu
|
||||||
|
eval/tofu_metrics/../../collator@eval.tofu.metrics.extraction_strength.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.extraction_strength.datasets: TOFU_QA_forget
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.privleak.pre_compute.mia_min_k: mia_min_k
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.privleak.pre_compute.mia_min_k.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.privleak.pre_compute.mia_min_k.datasets: TOFU_MIA
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio: wf_Truth_Ratio
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_PERT_Prob.datasets
|
||||||
|
: TOFU_QA_wf_pert
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Truth_Ratio.pre_compute.wf_Q_A_Prob.datasets
|
||||||
|
: TOFU_QA_wf
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE: wf_Q_A_ROUGE
|
||||||
|
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.generation_args: default
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_ROUGE.datasets: TOFU_QA_wf
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised: wf_Q_A_Prob_normalised
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob: wf_Q_A_PERT_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_PERT_Prob.datasets
|
||||||
|
: TOFU_QA_wf_pert
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob: wf_Q_A_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.wf_Q_A_Prob_normalised.pre_compute.wf_Q_A_Prob.datasets
|
||||||
|
: TOFU_QA_wf
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio: ra_Truth_Ratio
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_PERT_Prob.datasets
|
||||||
|
: TOFU_QA_ra_pert
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Truth_Ratio.pre_compute.ra_Q_A_Prob.datasets
|
||||||
|
: TOFU_QA_ra
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE: ra_Q_A_ROUGE
|
||||||
|
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.generation_args: default
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_ROUGE.datasets: TOFU_QA_ra
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised: ra_Q_A_Prob_normalised
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob: ra_Q_A_PERT_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_PERT_Prob.datasets
|
||||||
|
: TOFU_QA_ra_pert
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob: ra_Q_A_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.ra_Q_A_Prob_normalised.pre_compute.ra_Q_A_Prob.datasets
|
||||||
|
: TOFU_QA_ra
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio: retain_Truth_Ratio
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob: retain_Q_A_PERT_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PERT_Prob.datasets
|
||||||
|
: TOFU_QA_retain_pert
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob: retain_Q_A_PARA_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Truth_Ratio.pre_compute.retain_Q_A_PARA_Prob.datasets
|
||||||
|
: TOFU_QA_retain_para
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE: retain_Q_A_ROUGE
|
||||||
|
eval/tofu_metrics/./../../generation@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.generation_args: default
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_ROUGE.datasets: TOFU_QA_retain_eval
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob: retain_Q_A_Prob
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.model_utility.pre_compute.retain_Q_A_Prob.datasets: TOFU_QA_retain_eval
|
||||||
|
eval/tofu_metrics/../../generation@eval.tofu.metrics.forget_Q_A_ROUGE.generation_args: default
|
||||||
|
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_ROUGE.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_ROUGE.datasets: TOFU_QA_forget
|
||||||
|
eval/tofu_metrics/../../collator@eval.tofu.metrics.forget_Q_A_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/../../data/datasets@eval.tofu.metrics.forget_Q_A_Prob.datasets: TOFU_QA_forget
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio: forget_Truth_Ratio
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets
|
||||||
|
: TOFU_QA_forget_pert
|
||||||
|
eval/tofu_metrics/./.@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
|
||||||
|
? eval/tofu_metrics/././../../collator@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators
|
||||||
|
: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
? eval/tofu_metrics/././../../data/datasets@eval.tofu.metrics.forget_quality.pre_compute.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets
|
||||||
|
: TOFU_QA_forget_para
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob: forget_Q_A_PERT_Prob
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PERT_Prob.datasets: TOFU_QA_forget_pert
|
||||||
|
eval/tofu_metrics/.@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob: forget_Q_A_PARA_Prob
|
||||||
|
eval/tofu_metrics/./../../collator@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.collators: DataCollatorForSupervisedDatasetwithIndex
|
||||||
|
eval/tofu_metrics/./../../data/datasets@eval.tofu.metrics.forget_truth_ratio.pre_compute.forget_Q_A_PARA_Prob.datasets: TOFU_QA_forget_para
|
||||||
|
collator: DataCollatorForSupervisedDataset
|
||||||
|
data: unlearn
|
||||||
|
data/datasets@data.eval: null
|
||||||
|
data/datasets@data.retain: TOFU_QA_retain
|
||||||
|
data/datasets@data.forget: TOFU_QA_forget
|
||||||
|
trainer: RMU
|
||||||
|
model: Llama-3.2-1B-Instruct
|
||||||
|
hydra/env: default
|
||||||
|
hydra/callbacks: null
|
||||||
|
hydra/job_logging: colorlog
|
||||||
|
hydra/hydra_logging: colorlog
|
||||||
|
hydra/hydra_help: default
|
||||||
|
hydra/help: default
|
||||||
|
hydra/sweeper: basic
|
||||||
|
hydra/launcher: basic
|
||||||
|
hydra/output: default
|
||||||
|
verbose: false
|
||||||
21
.hydra/overrides.yaml
Normal file
21
.hydra/overrides.yaml
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
- experiment=unlearn/tofu/default.yaml
|
||||||
|
- trainer=RMU
|
||||||
|
- task_name=tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
- model=Llama-3.2-1B-Instruct
|
||||||
|
- forget_split=forget10
|
||||||
|
- retain_split=retain90
|
||||||
|
- holdout_split=holdout10
|
||||||
|
- model.model_args.pretrained_model_name_or_path=open-unlearning/tofu_Llama-3.2-1B-Instruct_full
|
||||||
|
- model.model_args.attn_implementation=sdpa
|
||||||
|
- trainer.args.per_device_train_batch_size=4
|
||||||
|
- trainer.args.gradient_accumulation_steps=4
|
||||||
|
- trainer.args.ddp_find_unused_parameters=true
|
||||||
|
- trainer.args.do_eval=false
|
||||||
|
- trainer.args.save_strategy=no
|
||||||
|
- +trainer.args.push_to_hub=true
|
||||||
|
- +trainer.args.hub_model_id=Jeesup/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
- +trainer.args.hub_strategy=end
|
||||||
|
- +trainer.args.hub_private_repo=false
|
||||||
|
- paths.output_dir=saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
- +quant=qat_w4
|
||||||
|
- trainer.method_args.module_regex=model\.layers\.3
|
||||||
56
README.md
Normal file
56
README.md
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
---
|
||||||
|
library_name: transformers
|
||||||
|
license: bsd-3-clause
|
||||||
|
base_model: open-unlearning/tofu_Llama-3.2-1B-Instruct_full
|
||||||
|
tags:
|
||||||
|
- generated_from_trainer
|
||||||
|
model-index:
|
||||||
|
- name: tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
results: []
|
||||||
|
---
|
||||||
|
|
||||||
|
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
||||||
|
should probably proofread and complete it, then remove this comment. -->
|
||||||
|
|
||||||
|
# tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
|
||||||
|
This model is a fine-tuned version of [open-unlearning/tofu_Llama-3.2-1B-Instruct_full](https://huggingface.co/open-unlearning/tofu_Llama-3.2-1B-Instruct_full) on an unknown dataset.
|
||||||
|
|
||||||
|
## Model description
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Intended uses & limitations
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Training and evaluation data
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Training procedure
|
||||||
|
|
||||||
|
### Training hyperparameters
|
||||||
|
|
||||||
|
The following hyperparameters were used during training:
|
||||||
|
- learning_rate: 1e-05
|
||||||
|
- train_batch_size: 4
|
||||||
|
- eval_batch_size: 16
|
||||||
|
- seed: 0
|
||||||
|
- gradient_accumulation_steps: 4
|
||||||
|
- total_train_batch_size: 16
|
||||||
|
- optimizer: Use OptimizerNames.PAGED_ADAMW with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
|
||||||
|
- lr_scheduler_type: linear
|
||||||
|
- lr_scheduler_warmup_steps: 25
|
||||||
|
- num_epochs: 10
|
||||||
|
|
||||||
|
### Training results
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- Transformers 4.51.3
|
||||||
|
- Pytorch 2.11.0+cu128
|
||||||
|
- Datasets 3.0.1
|
||||||
|
- Tokenizers 0.21.4
|
||||||
465
RMU.log
Normal file
465
RMU.log
Normal file
@@ -0,0 +1,465 @@
|
|||||||
|
[2026-05-21 11:57:33,698][model][INFO] - Setting pad_token as eos token: <|eot_id|>
|
||||||
|
[2026-05-21 11:57:37,813][evaluator][INFO] - Evaluations stored in the experiment directory: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
[2026-05-21 11:57:39,435][trainer][INFO] - RMU Trainer loaded, output_dir: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4
|
||||||
|
[2026-05-21 11:57:40,023][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 11:57:40,023][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-0/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 11:57:40,023][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-0/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 11:57:41,506][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 11:57:58,264][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 11:58:38,873][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 11:58:38,874][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6448834990772232
|
||||||
|
[2026-05-21 11:58:38,880][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 11:58:38,881][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 11:58:38,881][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 11:58:38,881][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 11:58:40,563][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 11:58:46,744][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.10887884757248685
|
||||||
|
[2026-05-21 11:58:48,505][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:02:17,635][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.27890823225307393
|
||||||
|
[2026-05-21 12:02:19,330][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 12:02:27,753][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:04:33,587][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:04:41,251][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:05:12,905][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 12:05:14,652][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 12:05:18,181][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:05:21,757][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:05:23,000][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:06:07,138][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:06:07,139][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:06:07,139][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 12:06:08,822][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 12:06:11,152][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:06:15,810][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:06:17,088][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:07:14,539][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:07:14,539][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:07:14,539][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 12:07:14,540][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 12:07:14,544][evaluator][INFO] - Result for metric model_utility: 0.2185219733034594
|
||||||
|
[2026-05-21 12:07:17,574][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 12:07:21,214][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 12:07:21,214][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 12:07:21,214][evaluator][INFO] - Result for metric privleak: -47.96999999040602
|
||||||
|
[2026-05-21 12:07:22,668][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 12:07:23,965][evaluator][INFO] - Result for metric extraction_strength: 0.05119585323242784
|
||||||
|
[2026-05-21 12:08:53,751][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 12:08:53,751][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-25/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 12:08:53,751][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-25/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 12:08:55,728][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:09:02,060][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:09:26,356][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 12:09:26,357][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6614321776157142
|
||||||
|
[2026-05-21 12:09:26,364][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 12:09:26,364][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 12:09:26,364][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 12:09:26,364][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 12:09:28,002][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 12:09:33,162][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.044040033797500655
|
||||||
|
[2026-05-21 12:09:34,852][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:11:29,404][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.19825707891118127
|
||||||
|
[2026-05-21 12:11:31,096][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 12:11:39,019][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:13:45,155][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:13:52,720][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:14:21,986][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 12:14:23,693][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 12:14:26,349][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:14:30,389][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:14:31,749][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:15:44,420][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:15:44,421][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:15:44,421][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 12:15:46,154][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 12:15:48,537][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:15:51,980][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:15:53,337][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:16:22,460][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:16:22,460][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:16:22,460][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 12:16:22,461][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 12:16:22,461][evaluator][INFO] - Result for metric model_utility: 0.08605862105946929
|
||||||
|
[2026-05-21 12:16:25,761][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 12:16:31,351][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 12:16:31,351][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 12:16:31,351][evaluator][INFO] - Result for metric privleak: -22.12749999557451
|
||||||
|
[2026-05-21 12:16:33,075][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 12:16:42,339][evaluator][INFO] - Result for metric extraction_strength: 0.04014118794649353
|
||||||
|
[2026-05-21 12:17:06,543][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 12:17:06,543][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-50/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 12:17:06,544][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-50/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 12:17:08,209][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:17:16,386][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:17:44,453][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 12:17:44,454][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6646114327873448
|
||||||
|
[2026-05-21 12:17:44,460][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 12:17:44,461][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 12:17:44,461][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 12:17:44,461][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 12:17:46,123][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 12:17:51,886][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.028171376134705498
|
||||||
|
[2026-05-21 12:17:53,595][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:19:56,169][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.18136216120754584
|
||||||
|
[2026-05-21 12:19:58,075][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 12:20:05,461][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:22:11,777][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:22:19,434][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:22:48,615][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 12:22:50,385][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 12:22:53,191][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:22:57,341][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:22:58,995][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:23:38,005][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:23:38,005][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:23:38,005][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 12:23:39,646][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 12:23:42,040][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:23:45,003][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:23:46,257][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:24:23,368][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:24:23,369][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:24:23,369][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 12:24:23,369][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 12:24:23,370][evaluator][INFO] - Result for metric model_utility: 0.07766678967490327
|
||||||
|
[2026-05-21 12:24:26,397][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 12:24:31,922][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 12:24:31,923][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 12:24:31,923][evaluator][INFO] - Result for metric privleak: -16.837499996632495
|
||||||
|
[2026-05-21 12:24:33,630][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 12:24:42,580][evaluator][INFO] - Result for metric extraction_strength: 0.03555271291935598
|
||||||
|
[2026-05-21 12:25:09,643][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 12:25:09,643][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-75/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 12:25:09,643][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-75/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 12:25:11,296][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:25:19,068][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:25:36,970][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 12:25:36,971][evaluator][INFO] - Result for metric forget_truth_ratio: 0.666956633608444
|
||||||
|
[2026-05-21 12:25:36,977][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 12:25:36,978][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 12:25:36,978][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 12:25:36,978][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 12:25:38,613][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 12:25:44,827][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.025599835189350415
|
||||||
|
[2026-05-21 12:25:46,467][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:27:48,567][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.17665323136017774
|
||||||
|
[2026-05-21 12:27:50,319][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 12:27:58,030][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:30:05,340][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:30:12,963][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:30:39,143][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 12:30:40,835][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 12:30:43,488][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:30:47,643][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:30:48,950][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:31:28,106][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:31:28,107][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:31:28,107][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 12:31:29,859][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 12:31:32,220][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:31:35,651][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:31:36,902][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:32:16,255][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:32:16,256][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:32:16,256][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 12:32:16,256][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 12:32:16,257][evaluator][INFO] - Result for metric model_utility: 0.025169849004668194
|
||||||
|
[2026-05-21 12:32:19,201][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 12:32:24,841][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 12:32:24,841][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 12:32:24,841][evaluator][INFO] - Result for metric privleak: -15.60999999687799
|
||||||
|
[2026-05-21 12:32:26,598][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 12:32:30,837][evaluator][INFO] - Result for metric extraction_strength: 0.03542379317218397
|
||||||
|
[2026-05-21 12:32:55,301][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 12:32:55,301][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-100/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 12:32:55,301][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-100/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 12:32:56,927][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:33:02,940][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:33:32,151][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 12:33:32,152][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6650891487506957
|
||||||
|
[2026-05-21 12:33:32,158][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 12:33:32,159][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 12:33:32,159][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 12:33:32,159][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 12:33:33,843][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 12:33:38,385][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.024368888099124887
|
||||||
|
[2026-05-21 12:33:39,662][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:35:08,076][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.17340090189465512
|
||||||
|
[2026-05-21 12:35:09,710][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 12:35:14,668][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:37:19,876][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:37:27,741][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:37:55,967][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 12:37:57,746][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 12:38:00,481][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:38:04,623][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:38:06,064][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:38:40,084][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:38:40,084][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:38:40,084][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 12:38:41,711][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 12:38:44,062][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:38:47,631][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:38:48,878][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:39:28,314][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:39:28,314][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:39:28,314][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 12:39:28,315][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 12:39:28,315][evaluator][INFO] - Result for metric model_utility: 0.07572111776600261
|
||||||
|
[2026-05-21 12:39:31,238][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 12:39:36,722][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 12:39:36,722][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 12:39:36,722][evaluator][INFO] - Result for metric privleak: -15.054999996988997
|
||||||
|
[2026-05-21 12:39:38,473][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 12:39:45,286][evaluator][INFO] - Result for metric extraction_strength: 0.034881082155764
|
||||||
|
[2026-05-21 12:40:09,613][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 12:40:09,613][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-125/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 12:40:09,613][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-125/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 12:40:11,550][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:40:19,424][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:40:49,178][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 12:40:49,179][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6653264816436281
|
||||||
|
[2026-05-21 12:40:49,185][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 12:40:49,186][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 12:40:49,186][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 12:40:49,186][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 12:40:50,857][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 12:40:56,868][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.023090821445148322
|
||||||
|
[2026-05-21 12:40:58,563][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:42:59,770][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.17290647721627245
|
||||||
|
[2026-05-21 12:43:01,413][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 12:43:08,531][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:44:48,920][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:44:56,881][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:45:26,437][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 12:45:28,234][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 12:45:31,003][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:45:35,107][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:45:36,368][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:46:14,962][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:46:14,962][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:46:14,962][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 12:46:16,815][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 12:46:19,204][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:46:21,563][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:46:22,945][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:46:59,910][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:46:59,911][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:46:59,911][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 12:46:59,911][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 12:46:59,912][evaluator][INFO] - Result for metric model_utility: 0.061282254233379956
|
||||||
|
[2026-05-21 12:47:02,933][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 12:47:08,271][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 12:47:08,271][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 12:47:08,271][evaluator][INFO] - Result for metric privleak: -14.518749997096252
|
||||||
|
[2026-05-21 12:47:10,069][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 12:47:32,061][evaluator][INFO] - Result for metric extraction_strength: 0.03492969775585019
|
||||||
|
[2026-05-21 12:47:52,836][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 12:47:52,836][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-150/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 12:47:52,836][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-150/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 12:47:54,493][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:48:02,146][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:48:31,769][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 12:48:31,770][evaluator][INFO] - Result for metric forget_truth_ratio: 0.665408632731608
|
||||||
|
[2026-05-21 12:48:31,776][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 12:48:31,777][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 12:48:31,777][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 12:48:31,777][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 12:48:33,422][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 12:48:37,862][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.022637412079275235
|
||||||
|
[2026-05-21 12:48:39,116][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:50:46,079][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.16969368902404242
|
||||||
|
[2026-05-21 12:50:48,146][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 12:50:54,741][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:52:56,349][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:53:02,778][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:53:20,514][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 12:53:22,173][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 12:53:25,211][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:53:29,602][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:53:31,058][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:54:09,744][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:54:09,745][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:54:09,745][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 12:54:11,393][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 12:54:13,787][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:54:17,260][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 12:54:18,762][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:54:58,987][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:54:58,988][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 12:54:58,988][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 12:54:58,988][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 12:54:58,989][evaluator][INFO] - Result for metric model_utility: 0.07293884758887383
|
||||||
|
[2026-05-21 12:55:01,964][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 12:55:04,458][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 12:55:04,458][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 12:55:04,458][evaluator][INFO] - Result for metric privleak: -14.298749997140254
|
||||||
|
[2026-05-21 12:55:06,059][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 12:55:09,527][evaluator][INFO] - Result for metric extraction_strength: 0.034710650136802565
|
||||||
|
[2026-05-21 12:55:31,076][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 12:55:31,076][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-175/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 12:55:31,076][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-175/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 12:55:32,704][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 12:55:38,755][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 12:56:06,854][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 12:56:06,855][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6661571899518759
|
||||||
|
[2026-05-21 12:56:06,861][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 12:56:06,862][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 12:56:06,862][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 12:56:06,862][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 12:56:08,508][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 12:56:14,367][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.022689154595791478
|
||||||
|
[2026-05-21 12:56:16,040][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 12:58:20,768][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.17071292253126477
|
||||||
|
[2026-05-21 12:58:22,468][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 12:58:29,886][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:00:37,081][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 13:00:43,094][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:01:10,854][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 13:01:12,568][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 13:01:15,228][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:01:19,302][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 13:01:20,611][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:01:54,745][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:01:54,745][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:01:54,745][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 13:01:56,526][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 13:01:58,924][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:02:02,375][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 13:02:03,624][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:02:27,784][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:02:27,784][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:02:27,784][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 13:02:27,784][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 13:02:27,785][evaluator][INFO] - Result for metric model_utility: 0.024938334488339972
|
||||||
|
[2026-05-21 13:02:30,834][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 13:02:35,356][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 13:02:35,356][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 13:02:35,356][evaluator][INFO] - Result for metric privleak: -14.50124999709975
|
||||||
|
[2026-05-21 13:02:36,768][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 13:02:52,494][evaluator][INFO] - Result for metric extraction_strength: 0.03454297434412678
|
||||||
|
[2026-05-21 13:03:16,128][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 13:03:16,129][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-200/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 13:03:16,129][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-200/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 13:03:17,801][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 13:03:25,631][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:03:56,057][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 13:03:56,058][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6657672475603854
|
||||||
|
[2026-05-21 13:03:56,064][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 13:03:56,065][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 13:03:56,065][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 13:03:56,065][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 13:03:57,750][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 13:04:02,208][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.022478839480972967
|
||||||
|
[2026-05-21 13:04:03,476][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:06:08,123][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.17469059451335844
|
||||||
|
[2026-05-21 13:06:09,777][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 13:06:17,186][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:08:23,471][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 13:08:31,153][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:09:00,265][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 13:09:01,944][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 13:09:04,573][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:09:07,999][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 13:09:09,674][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:09:45,248][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:09:45,249][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:09:45,249][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 13:09:46,946][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 13:09:49,056][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:09:52,602][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 13:09:54,048][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:10:30,231][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:10:30,231][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:10:30,231][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 13:10:30,232][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 13:10:30,232][evaluator][INFO] - Result for metric model_utility: 0.02489956345780598
|
||||||
|
[2026-05-21 13:10:33,282][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 13:10:36,848][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 13:10:36,849][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 13:10:36,849][evaluator][INFO] - Result for metric privleak: -13.848749997230259
|
||||||
|
[2026-05-21 13:10:38,234][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 13:10:56,184][evaluator][INFO] - Result for metric extraction_strength: 0.03481065013680257
|
||||||
|
[2026-05-21 13:11:08,088][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 13:11:08,088][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-225/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 13:11:08,088][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-225/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 13:11:09,829][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 13:11:17,751][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:11:46,542][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 13:11:46,543][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6661828793220076
|
||||||
|
[2026-05-21 13:11:46,550][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 13:11:46,550][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 13:11:46,550][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 13:11:46,550][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 13:11:48,331][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 13:11:54,570][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.02249195015472651
|
||||||
|
[2026-05-21 13:11:56,226][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:13:55,029][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.16997095669432055
|
||||||
|
[2026-05-21 13:13:56,746][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 13:14:04,157][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:16:10,552][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 13:16:18,183][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:16:47,385][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 13:16:49,063][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 13:16:51,612][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:16:55,772][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 13:16:57,083][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:17:36,254][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:17:36,254][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:17:36,254][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 13:17:37,889][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 13:17:40,234][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:17:43,663][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 13:17:45,073][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:18:21,334][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:18:21,334][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:18:21,334][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 13:18:21,335][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 13:18:21,335][evaluator][INFO] - Result for metric model_utility: 0.034445471917355024
|
||||||
|
[2026-05-21 13:18:24,581][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 13:18:30,118][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 13:18:30,118][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 13:18:30,118][evaluator][INFO] - Result for metric privleak: -13.99624999720075
|
||||||
|
[2026-05-21 13:18:31,791][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 13:18:33,441][evaluator][INFO] - Result for metric extraction_strength: 0.03500545533160776
|
||||||
|
[2026-05-21 13:18:48,997][evaluator][INFO] - ***** Running TOFU evaluation suite *****
|
||||||
|
[2026-05-21 13:18:48,998][evaluator][INFO] - Fine-grained evaluations will be saved to: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-250/evals/TOFU_EVAL.json
|
||||||
|
[2026-05-21 13:18:48,998][evaluator][INFO] - Aggregated evaluations will be summarised in: saves/unlearn/qat-baseline/tofu_Llama-3.2-1B-Instruct_forget10_RMU_qat-int4/checkpoint-250/evals/TOFU_SUMMARY.json
|
||||||
|
[2026-05-21 13:18:50,692][metrics][INFO] - Evaluating forget_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 13:18:58,986][metrics][INFO] - Evaluating forget_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:19:25,257][metrics][INFO] - Evaluating forget_truth_ratio
|
||||||
|
[2026-05-21 13:19:25,258][evaluator][INFO] - Result for metric forget_truth_ratio: 0.6682139209275343
|
||||||
|
[2026-05-21 13:19:25,264][metrics][INFO] - Skipping forget_quality's precompute forget_truth_ratio, already evaluated.
|
||||||
|
[2026-05-21 13:19:25,265][metrics][INFO] - Evaluating forget_quality
|
||||||
|
[2026-05-21 13:19:25,265][metrics][WARNING] - retain_model_logs not provided in reference_logs, setting forget_quality to None
|
||||||
|
[2026-05-21 13:19:25,265][evaluator][INFO] - Result for metric forget_quality: None
|
||||||
|
[2026-05-21 13:19:26,973][metrics][INFO] - Evaluating forget_Q_A_Prob
|
||||||
|
[2026-05-21 13:19:32,982][evaluator][INFO] - Result for metric forget_Q_A_Prob: 0.022312334740927326
|
||||||
|
[2026-05-21 13:19:34,673][metrics][INFO] - Evaluating forget_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:21:16,645][evaluator][INFO] - Result for metric forget_Q_A_ROUGE: 0.16821955009936282
|
||||||
|
[2026-05-21 13:21:18,356][metrics][INFO] - Evaluating retain_Q_A_Prob
|
||||||
|
[2026-05-21 13:21:25,924][metrics][INFO] - Evaluating retain_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:23:30,694][metrics][INFO] - Evaluating retain_Q_A_PARA_Prob
|
||||||
|
[2026-05-21 13:23:38,159][metrics][INFO] - Evaluating retain_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:24:07,357][metrics][INFO] - Evaluating retain_Truth_Ratio
|
||||||
|
[2026-05-21 13:24:09,157][metrics][INFO] - Evaluating ra_Q_A_Prob
|
||||||
|
[2026-05-21 13:24:11,661][metrics][INFO] - Evaluating ra_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:24:15,807][metrics][INFO] - Evaluating ra_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 13:24:17,063][metrics][INFO] - Evaluating ra_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:24:56,249][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:24:56,250][metrics][INFO] - Skipping ra_Truth_Ratio's precompute ra_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:24:56,250][metrics][INFO] - Evaluating ra_Truth_Ratio
|
||||||
|
[2026-05-21 13:24:57,929][metrics][INFO] - Evaluating wf_Q_A_Prob
|
||||||
|
[2026-05-21 13:25:00,268][metrics][INFO] - Evaluating wf_Q_A_PERT_Prob
|
||||||
|
[2026-05-21 13:25:03,792][metrics][INFO] - Evaluating wf_Q_A_Prob_normalised
|
||||||
|
[2026-05-21 13:25:05,243][metrics][INFO] - Evaluating wf_Q_A_ROUGE
|
||||||
|
[2026-05-21 13:25:41,021][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:25:41,021][metrics][INFO] - Skipping wf_Truth_Ratio's precompute wf_Q_A_PERT_Prob, already evaluated.
|
||||||
|
[2026-05-21 13:25:41,021][metrics][INFO] - Evaluating wf_Truth_Ratio
|
||||||
|
[2026-05-21 13:25:41,021][metrics][INFO] - Evaluating model_utility
|
||||||
|
[2026-05-21 13:25:41,022][evaluator][INFO] - Result for metric model_utility: 0.024945798930326533
|
||||||
|
[2026-05-21 13:25:44,127][metrics][INFO] - Evaluating mia_min_k
|
||||||
|
[2026-05-21 13:25:49,445][metrics][INFO] - Evaluating privleak
|
||||||
|
[2026-05-21 13:25:49,445][metrics][WARNING] - retain_model_logs evals not provided for privleak, using default retain auc of 0.5
|
||||||
|
[2026-05-21 13:25:49,445][evaluator][INFO] - Result for metric privleak: -14.171249997165742
|
||||||
|
[2026-05-21 13:25:52,550][metrics][INFO] - Evaluating extraction_strength
|
||||||
|
[2026-05-21 13:26:14,618][evaluator][INFO] - Result for metric extraction_strength: 0.034667360093512525
|
||||||
39
config.json
Normal file
39
config.json
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"LlamaForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": false,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 128000,
|
||||||
|
"eos_token_id": [
|
||||||
|
128001,
|
||||||
|
128008,
|
||||||
|
128009
|
||||||
|
],
|
||||||
|
"head_dim": 64,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 2048,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 8192,
|
||||||
|
"max_position_embeddings": 131072,
|
||||||
|
"mlp_bias": false,
|
||||||
|
"model_type": "llama",
|
||||||
|
"num_attention_heads": 32,
|
||||||
|
"num_hidden_layers": 16,
|
||||||
|
"num_key_value_heads": 8,
|
||||||
|
"pretraining_tp": 1,
|
||||||
|
"rms_norm_eps": 1e-05,
|
||||||
|
"rope_scaling": {
|
||||||
|
"factor": 32.0,
|
||||||
|
"high_freq_factor": 4.0,
|
||||||
|
"low_freq_factor": 1.0,
|
||||||
|
"original_max_position_embeddings": 8192,
|
||||||
|
"rope_type": "llama3"
|
||||||
|
},
|
||||||
|
"rope_theta": 500000.0,
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"torch_dtype": "bfloat16",
|
||||||
|
"transformers_version": "4.51.3",
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 128256
|
||||||
|
}
|
||||||
12
generation_config.json
Normal file
12
generation_config.json
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"bos_token_id": 128000,
|
||||||
|
"do_sample": true,
|
||||||
|
"eos_token_id": [
|
||||||
|
128001,
|
||||||
|
128008,
|
||||||
|
128009
|
||||||
|
],
|
||||||
|
"temperature": 0.6,
|
||||||
|
"top_p": 0.9,
|
||||||
|
"transformers_version": "4.51.3"
|
||||||
|
}
|
||||||
3
logs/events.out.tfevents.1779332260.jeesup.2790209.0
Normal file
3
logs/events.out.tfevents.1779332260.jeesup.2790209.0
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:e890fdd40ced5857881b87c9686bd84699393fe0b5adc51c418731d10f800f12
|
||||||
|
size 20641
|
||||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:9fd74f8f3d05bbe00721eec97719405626cf0b66353316d9214c03bf26635df1
|
||||||
|
size 2471645608
|
||||||
17
special_tokens_map.json
Normal file
17
special_tokens_map.json
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"bos_token": {
|
||||||
|
"content": "<|begin_of_text|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"eos_token": {
|
||||||
|
"content": "<|eot_id|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"pad_token": "<|eot_id|>"
|
||||||
|
}
|
||||||
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
Binary file not shown.
2064
tokenizer_config.json
Normal file
2064
tokenizer_config.json
Normal file
File diff suppressed because it is too large
Load Diff
503
trainer_state.json
Normal file
503
trainer_state.json
Normal file
@@ -0,0 +1,503 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 10.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 250,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0,
|
||||||
|
"extraction_strength": 0.05119585323242784,
|
||||||
|
"forget_Q_A_Prob": 0.10887884757248685,
|
||||||
|
"forget_Q_A_ROUGE": 0.27890823225307393,
|
||||||
|
"forget_truth_ratio": 0.6448834990772232,
|
||||||
|
"model_utility": 0.2185219733034594,
|
||||||
|
"privleak": -47.96999999040602,
|
||||||
|
"step": 0
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2,
|
||||||
|
"grad_norm": 0.33203125,
|
||||||
|
"learning_rate": 1.6000000000000001e-06,
|
||||||
|
"loss": 0.0302,
|
||||||
|
"step": 5
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4,
|
||||||
|
"grad_norm": 0.31640625,
|
||||||
|
"learning_rate": 3.6000000000000003e-06,
|
||||||
|
"loss": 0.0295,
|
||||||
|
"step": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6,
|
||||||
|
"grad_norm": 0.2890625,
|
||||||
|
"learning_rate": 5.600000000000001e-06,
|
||||||
|
"loss": 0.0286,
|
||||||
|
"step": 15
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8,
|
||||||
|
"grad_norm": 0.3203125,
|
||||||
|
"learning_rate": 7.600000000000001e-06,
|
||||||
|
"loss": 0.0293,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"grad_norm": 0.1494140625,
|
||||||
|
"learning_rate": 9.600000000000001e-06,
|
||||||
|
"loss": 0.0271,
|
||||||
|
"step": 25
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"extraction_strength": 0.04014118794649353,
|
||||||
|
"forget_Q_A_Prob": 0.044040033797500655,
|
||||||
|
"forget_Q_A_ROUGE": 0.19825707891118127,
|
||||||
|
"forget_truth_ratio": 0.6614321776157142,
|
||||||
|
"model_utility": 0.08605862105946929,
|
||||||
|
"privleak": -22.12749999557451,
|
||||||
|
"step": 25
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2,
|
||||||
|
"grad_norm": 0.11083984375,
|
||||||
|
"learning_rate": 9.822222222222223e-06,
|
||||||
|
"loss": 0.0254,
|
||||||
|
"step": 30
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.4,
|
||||||
|
"grad_norm": 0.10400390625,
|
||||||
|
"learning_rate": 9.600000000000001e-06,
|
||||||
|
"loss": 0.0254,
|
||||||
|
"step": 35
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.6,
|
||||||
|
"grad_norm": 0.08984375,
|
||||||
|
"learning_rate": 9.377777777777779e-06,
|
||||||
|
"loss": 0.024,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.8,
|
||||||
|
"grad_norm": 0.10205078125,
|
||||||
|
"learning_rate": 9.155555555555557e-06,
|
||||||
|
"loss": 0.0246,
|
||||||
|
"step": 45
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"grad_norm": 0.08544921875,
|
||||||
|
"learning_rate": 8.933333333333333e-06,
|
||||||
|
"loss": 0.0241,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.0,
|
||||||
|
"extraction_strength": 0.03555271291935598,
|
||||||
|
"forget_Q_A_Prob": 0.028171376134705498,
|
||||||
|
"forget_Q_A_ROUGE": 0.18136216120754584,
|
||||||
|
"forget_truth_ratio": 0.6646114327873448,
|
||||||
|
"model_utility": 0.07766678967490327,
|
||||||
|
"privleak": -16.837499996632495,
|
||||||
|
"step": 50
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.2,
|
||||||
|
"grad_norm": 0.087890625,
|
||||||
|
"learning_rate": 8.711111111111111e-06,
|
||||||
|
"loss": 0.0242,
|
||||||
|
"step": 55
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.4,
|
||||||
|
"grad_norm": 0.10595703125,
|
||||||
|
"learning_rate": 8.48888888888889e-06,
|
||||||
|
"loss": 0.0243,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.6,
|
||||||
|
"grad_norm": 0.09033203125,
|
||||||
|
"learning_rate": 8.266666666666667e-06,
|
||||||
|
"loss": 0.024,
|
||||||
|
"step": 65
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 2.8,
|
||||||
|
"grad_norm": 0.0859375,
|
||||||
|
"learning_rate": 8.044444444444444e-06,
|
||||||
|
"loss": 0.0236,
|
||||||
|
"step": 70
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"grad_norm": 0.0791015625,
|
||||||
|
"learning_rate": 7.822222222222224e-06,
|
||||||
|
"loss": 0.0228,
|
||||||
|
"step": 75
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.0,
|
||||||
|
"extraction_strength": 0.03542379317218397,
|
||||||
|
"forget_Q_A_Prob": 0.025599835189350415,
|
||||||
|
"forget_Q_A_ROUGE": 0.17665323136017774,
|
||||||
|
"forget_truth_ratio": 0.666956633608444,
|
||||||
|
"model_utility": 0.025169849004668194,
|
||||||
|
"privleak": -15.60999999687799,
|
||||||
|
"step": 75
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.2,
|
||||||
|
"grad_norm": 0.083984375,
|
||||||
|
"learning_rate": 7.600000000000001e-06,
|
||||||
|
"loss": 0.023,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.4,
|
||||||
|
"grad_norm": 0.07958984375,
|
||||||
|
"learning_rate": 7.377777777777778e-06,
|
||||||
|
"loss": 0.0228,
|
||||||
|
"step": 85
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.6,
|
||||||
|
"grad_norm": 0.08544921875,
|
||||||
|
"learning_rate": 7.155555555555556e-06,
|
||||||
|
"loss": 0.0236,
|
||||||
|
"step": 90
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 3.8,
|
||||||
|
"grad_norm": 0.07763671875,
|
||||||
|
"learning_rate": 6.9333333333333344e-06,
|
||||||
|
"loss": 0.0235,
|
||||||
|
"step": 95
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.0,
|
||||||
|
"grad_norm": 0.08251953125,
|
||||||
|
"learning_rate": 6.711111111111111e-06,
|
||||||
|
"loss": 0.0233,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.0,
|
||||||
|
"extraction_strength": 0.034881082155764,
|
||||||
|
"forget_Q_A_Prob": 0.024368888099124887,
|
||||||
|
"forget_Q_A_ROUGE": 0.17340090189465512,
|
||||||
|
"forget_truth_ratio": 0.6650891487506957,
|
||||||
|
"model_utility": 0.07572111776600261,
|
||||||
|
"privleak": -15.054999996988997,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.2,
|
||||||
|
"grad_norm": 0.08544921875,
|
||||||
|
"learning_rate": 6.488888888888889e-06,
|
||||||
|
"loss": 0.0243,
|
||||||
|
"step": 105
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.4,
|
||||||
|
"grad_norm": 0.08154296875,
|
||||||
|
"learning_rate": 6.266666666666668e-06,
|
||||||
|
"loss": 0.0231,
|
||||||
|
"step": 110
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.6,
|
||||||
|
"grad_norm": 0.08056640625,
|
||||||
|
"learning_rate": 6.044444444444445e-06,
|
||||||
|
"loss": 0.0231,
|
||||||
|
"step": 115
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 4.8,
|
||||||
|
"grad_norm": 0.080078125,
|
||||||
|
"learning_rate": 5.822222222222223e-06,
|
||||||
|
"loss": 0.0231,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 5.0,
|
||||||
|
"grad_norm": 0.0810546875,
|
||||||
|
"learning_rate": 5.600000000000001e-06,
|
||||||
|
"loss": 0.0225,
|
||||||
|
"step": 125
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 5.0,
|
||||||
|
"extraction_strength": 0.03492969775585019,
|
||||||
|
"forget_Q_A_Prob": 0.023090821445148322,
|
||||||
|
"forget_Q_A_ROUGE": 0.17290647721627245,
|
||||||
|
"forget_truth_ratio": 0.6653264816436281,
|
||||||
|
"model_utility": 0.061282254233379956,
|
||||||
|
"privleak": -14.518749997096252,
|
||||||
|
"step": 125
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 5.2,
|
||||||
|
"grad_norm": 0.0810546875,
|
||||||
|
"learning_rate": 5.3777777777777784e-06,
|
||||||
|
"loss": 0.0235,
|
||||||
|
"step": 130
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 5.4,
|
||||||
|
"grad_norm": 0.08203125,
|
||||||
|
"learning_rate": 5.155555555555556e-06,
|
||||||
|
"loss": 0.0227,
|
||||||
|
"step": 135
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 5.6,
|
||||||
|
"grad_norm": 0.09619140625,
|
||||||
|
"learning_rate": 4.933333333333334e-06,
|
||||||
|
"loss": 0.0235,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 5.8,
|
||||||
|
"grad_norm": 0.0947265625,
|
||||||
|
"learning_rate": 4.711111111111111e-06,
|
||||||
|
"loss": 0.0245,
|
||||||
|
"step": 145
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 6.0,
|
||||||
|
"grad_norm": 0.0849609375,
|
||||||
|
"learning_rate": 4.488888888888889e-06,
|
||||||
|
"loss": 0.0225,
|
||||||
|
"step": 150
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 6.0,
|
||||||
|
"extraction_strength": 0.034710650136802565,
|
||||||
|
"forget_Q_A_Prob": 0.022637412079275235,
|
||||||
|
"forget_Q_A_ROUGE": 0.16969368902404242,
|
||||||
|
"forget_truth_ratio": 0.665408632731608,
|
||||||
|
"model_utility": 0.07293884758887383,
|
||||||
|
"privleak": -14.298749997140254,
|
||||||
|
"step": 150
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 6.2,
|
||||||
|
"grad_norm": 0.07861328125,
|
||||||
|
"learning_rate": 4.266666666666668e-06,
|
||||||
|
"loss": 0.0239,
|
||||||
|
"step": 155
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 6.4,
|
||||||
|
"grad_norm": 0.08447265625,
|
||||||
|
"learning_rate": 4.044444444444445e-06,
|
||||||
|
"loss": 0.0234,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 6.6,
|
||||||
|
"grad_norm": 0.0986328125,
|
||||||
|
"learning_rate": 3.8222222222222224e-06,
|
||||||
|
"loss": 0.0236,
|
||||||
|
"step": 165
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 6.8,
|
||||||
|
"grad_norm": 0.091796875,
|
||||||
|
"learning_rate": 3.6000000000000003e-06,
|
||||||
|
"loss": 0.023,
|
||||||
|
"step": 170
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 7.0,
|
||||||
|
"grad_norm": 0.0791015625,
|
||||||
|
"learning_rate": 3.377777777777778e-06,
|
||||||
|
"loss": 0.0231,
|
||||||
|
"step": 175
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 7.0,
|
||||||
|
"extraction_strength": 0.03454297434412678,
|
||||||
|
"forget_Q_A_Prob": 0.022689154595791478,
|
||||||
|
"forget_Q_A_ROUGE": 0.17071292253126477,
|
||||||
|
"forget_truth_ratio": 0.6661571899518759,
|
||||||
|
"model_utility": 0.024938334488339972,
|
||||||
|
"privleak": -14.50124999709975,
|
||||||
|
"step": 175
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 7.2,
|
||||||
|
"grad_norm": 0.0859375,
|
||||||
|
"learning_rate": 3.1555555555555555e-06,
|
||||||
|
"loss": 0.0234,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 7.4,
|
||||||
|
"grad_norm": 0.083984375,
|
||||||
|
"learning_rate": 2.9333333333333338e-06,
|
||||||
|
"loss": 0.0229,
|
||||||
|
"step": 185
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 7.6,
|
||||||
|
"grad_norm": 0.0849609375,
|
||||||
|
"learning_rate": 2.7111111111111116e-06,
|
||||||
|
"loss": 0.0231,
|
||||||
|
"step": 190
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 7.8,
|
||||||
|
"grad_norm": 0.080078125,
|
||||||
|
"learning_rate": 2.488888888888889e-06,
|
||||||
|
"loss": 0.0234,
|
||||||
|
"step": 195
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 8.0,
|
||||||
|
"grad_norm": 0.08056640625,
|
||||||
|
"learning_rate": 2.266666666666667e-06,
|
||||||
|
"loss": 0.023,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 8.0,
|
||||||
|
"extraction_strength": 0.03481065013680257,
|
||||||
|
"forget_Q_A_Prob": 0.022478839480972967,
|
||||||
|
"forget_Q_A_ROUGE": 0.17469059451335844,
|
||||||
|
"forget_truth_ratio": 0.6657672475603854,
|
||||||
|
"model_utility": 0.02489956345780598,
|
||||||
|
"privleak": -13.848749997230259,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 8.2,
|
||||||
|
"grad_norm": 0.08251953125,
|
||||||
|
"learning_rate": 2.0444444444444447e-06,
|
||||||
|
"loss": 0.0234,
|
||||||
|
"step": 205
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 8.4,
|
||||||
|
"grad_norm": 0.0771484375,
|
||||||
|
"learning_rate": 1.8222222222222225e-06,
|
||||||
|
"loss": 0.0243,
|
||||||
|
"step": 210
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 8.6,
|
||||||
|
"grad_norm": 0.0869140625,
|
||||||
|
"learning_rate": 1.6000000000000001e-06,
|
||||||
|
"loss": 0.0231,
|
||||||
|
"step": 215
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 8.8,
|
||||||
|
"grad_norm": 0.080078125,
|
||||||
|
"learning_rate": 1.377777777777778e-06,
|
||||||
|
"loss": 0.0227,
|
||||||
|
"step": 220
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 9.0,
|
||||||
|
"grad_norm": 0.07177734375,
|
||||||
|
"learning_rate": 1.1555555555555556e-06,
|
||||||
|
"loss": 0.0229,
|
||||||
|
"step": 225
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 9.0,
|
||||||
|
"extraction_strength": 0.03500545533160776,
|
||||||
|
"forget_Q_A_Prob": 0.02249195015472651,
|
||||||
|
"forget_Q_A_ROUGE": 0.16997095669432055,
|
||||||
|
"forget_truth_ratio": 0.6661828793220076,
|
||||||
|
"model_utility": 0.034445471917355024,
|
||||||
|
"privleak": -13.99624999720075,
|
||||||
|
"step": 225
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 9.2,
|
||||||
|
"grad_norm": 0.076171875,
|
||||||
|
"learning_rate": 9.333333333333334e-07,
|
||||||
|
"loss": 0.0223,
|
||||||
|
"step": 230
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 9.4,
|
||||||
|
"grad_norm": 0.0791015625,
|
||||||
|
"learning_rate": 7.111111111111112e-07,
|
||||||
|
"loss": 0.0232,
|
||||||
|
"step": 235
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 9.6,
|
||||||
|
"grad_norm": 0.0849609375,
|
||||||
|
"learning_rate": 4.88888888888889e-07,
|
||||||
|
"loss": 0.023,
|
||||||
|
"step": 240
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 9.8,
|
||||||
|
"grad_norm": 0.0849609375,
|
||||||
|
"learning_rate": 2.666666666666667e-07,
|
||||||
|
"loss": 0.0235,
|
||||||
|
"step": 245
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 10.0,
|
||||||
|
"grad_norm": 0.0830078125,
|
||||||
|
"learning_rate": 4.444444444444445e-08,
|
||||||
|
"loss": 0.0232,
|
||||||
|
"step": 250
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 10.0,
|
||||||
|
"extraction_strength": 0.034667360093512525,
|
||||||
|
"forget_Q_A_Prob": 0.022312334740927326,
|
||||||
|
"forget_Q_A_ROUGE": 0.16821955009936282,
|
||||||
|
"forget_truth_ratio": 0.6682139209275343,
|
||||||
|
"model_utility": 0.024945798930326533,
|
||||||
|
"privleak": -14.171249997165742,
|
||||||
|
"step": 250
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 10.0,
|
||||||
|
"step": 250,
|
||||||
|
"total_flos": 0.0,
|
||||||
|
"train_loss": 0.0240108642578125,
|
||||||
|
"train_runtime": 5314.6353,
|
||||||
|
"train_samples_per_second": 0.753,
|
||||||
|
"train_steps_per_second": 0.047
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 5,
|
||||||
|
"max_steps": 250,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 10,
|
||||||
|
"save_steps": 500,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": false,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 0.0,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:21cd2890c569286b2ba5b3dca58e45827bdb891c3f9981b3dfbc87551e9551ce
|
||||||
|
size 5841
|
||||||
Reference in New Issue
Block a user