51cf6da7cd
Model: Grogros/dm-llama3.2-1BI-LucieFr-Al4-OWT-TV-ablation-h2d4 Source: Original Platform
52 lines
1.1 KiB
YAML
52 lines
1.1 KiB
YAML
alpha: 0.1
|
|
base_model: meta-llama/Llama-3.2-1B-Instruct
|
|
custom_name: TV
|
|
dtype: bfloat16
|
|
lambdas:
|
|
- 1.0
|
|
- 1.0
|
|
- 1.0
|
|
lora_config: null
|
|
loss_types:
|
|
- anti-watermark-tv
|
|
- anti-watermark-tv
|
|
meta_learning_config: null
|
|
n_wm_tokens: 0
|
|
proportions:
|
|
- 0.6
|
|
- 0.2
|
|
- 0.2
|
|
random_training_config: null
|
|
regularization_datasets:
|
|
- !!python/object/apply:finetuning.dataset.DatasetType
|
|
- AlpacaGPT4
|
|
- !!python/object/apply:finetuning.dataset.DatasetType
|
|
- OpenWebText
|
|
sequence_length: 512
|
|
streaming: true
|
|
training_args:
|
|
bf16: false
|
|
do_train: true
|
|
fp16: false
|
|
gradient_accumulation_steps: 16
|
|
gradient_checkpointing: false
|
|
hub_strategy: all_checkpoints
|
|
learning_rate: 2.0e-05
|
|
logging_steps: 10
|
|
lr_scheduler_type: cosine
|
|
max_steps: 2500
|
|
num_train_epochs: 1
|
|
optim: adafactor
|
|
output_dir: Grogros/dm-llama3.2-1BI-LucieFr-Al4-OWT-TV-ablation-h2d4
|
|
overwrite_output_dir: true
|
|
per_device_train_batch_size: 4
|
|
push_to_hub: true
|
|
report_to: none
|
|
save_steps: 500
|
|
save_strategy: steps
|
|
warmup_ratio: 0.1
|
|
watermark_datasets:
|
|
- !!python/object/apply:finetuning.dataset.DatasetType
|
|
- LucieFr
|
|
watermark_eval_config: []
|