1748 lines
1.3 MiB
Plaintext
1748 lines
1.3 MiB
Plaintext
|
2026-04-18 09:13:06 - INFO - __main__ - Model parameters ModelArguments(base_model_revision=None, model_name_or_path='/scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-sft-hh-helpful-4xh200-batch-64-20260418-015332', model_revision='main', model_code_revision=None, torch_dtype='bfloat16', tokenizer_name_or_path=None, trust_remote_code=False, attn_implementation='flash_attention_2', use_peft=False, lora_r=16, lora_alpha=32, lora_dropout=0.05, lora_target_modules=None, lora_modules_to_save=None, load_in_8bit=False, load_in_4bit=False, bnb_4bit_quant_type='nf4', use_bnb_nested_quant=False, bnb_4bit_quant_storage='uint8')
|
|||
|
|
2026-04-18 09:13:06 - INFO - __main__ - Data parameters DataArguments(chat_template=None, dataset_mixer={'Anthropic/hh-rlhf': 1.0}, text_column='text', dataset_splits=['train', 'test'], dataset_configs=['helpful-base'], dataset_dir=None, preprocessing_num_workers=12, use_persistent_hf_cache=True, hf_cache_dir='/scratch/feng.yulu/dynamic-dpo-v4/hf/datasets', truncation_side=None, auto_insert_empty_system_msg=True, disable_thinking=False, preprocessing_log_samples=0, preprocessing_log_dir=None)
|
|||
|
|
2026-04-18 09:13:06 - INFO - __main__ - Training/evaluation parameters EpsilonDPOConfig(
|
|||
|
|
_n_gpu=1,
|
|||
|
|
accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None, 'use_configured_state': False},
|
|||
|
|
adafactor=False,
|
|||
|
|
adam_beta1=0.9,
|
|||
|
|
adam_beta2=0.999,
|
|||
|
|
adam_epsilon=1e-08,
|
|||
|
|
auto_find_batch_size=False,
|
|||
|
|
average_tokens_across_devices=False,
|
|||
|
|
batch_eval_metrics=False,
|
|||
|
|
beta=0.1,
|
|||
|
|
bf16=True,
|
|||
|
|
bf16_full_eval=False,
|
|||
|
|
data_seed=None,
|
|||
|
|
dataloader_drop_last=True,
|
|||
|
|
dataloader_num_workers=0,
|
|||
|
|
dataloader_persistent_workers=False,
|
|||
|
|
dataloader_pin_memory=True,
|
|||
|
|
dataloader_prefetch_factor=None,
|
|||
|
|
dataset_num_proc=12,
|
|||
|
|
ddp_backend=None,
|
|||
|
|
ddp_broadcast_buffers=None,
|
|||
|
|
ddp_bucket_cap_mb=None,
|
|||
|
|
ddp_find_unused_parameters=None,
|
|||
|
|
ddp_timeout=1800,
|
|||
|
|
debug=[],
|
|||
|
|
deepspeed=None,
|
|||
|
|
disable_dropout=True,
|
|||
|
|
disable_tqdm=False,
|
|||
|
|
do_eval=True,
|
|||
|
|
do_predict=False,
|
|||
|
|
do_train=False,
|
|||
|
|
epsilon=0.01,
|
|||
|
|
eval_accumulation_steps=None,
|
|||
|
|
eval_delay=0,
|
|||
|
|
eval_do_concat_batches=True,
|
|||
|
|
eval_on_start=False,
|
|||
|
|
eval_steps=100,
|
|||
|
|
eval_strategy=IntervalStrategy.STEPS,
|
|||
|
|
eval_use_gather_object=False,
|
|||
|
|
f_alpha_divergence_coef=1.0,
|
|||
|
|
f_divergence_type=FDivergenceType.REVERSE_KL,
|
|||
|
|
force_use_ref_model=False,
|
|||
|
|
fp16=False,
|
|||
|
|
fp16_backend=auto,
|
|||
|
|
fp16_full_eval=False,
|
|||
|
|
fp16_opt_level=O1,
|
|||
|
|
fsdp=[],
|
|||
|
|
fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
|
|||
|
|
fsdp_min_num_params=0,
|
|||
|
|
fsdp_transformer_layer_cls_to_wrap=None,
|
|||
|
|
full_determinism=False,
|
|||
|
|
generate_during_eval=False,
|
|||
|
|
gradient_accumulation_steps=2,
|
|||
|
|
gradient_checkpointing=True,
|
|||
|
|
gradient_checkpointing_kwargs={'use_reentrant': False},
|
|||
|
|
greater_is_better=None,
|
|||
|
|
group_by_length=False,
|
|||
|
|
half_precision_backend=auto,
|
|||
|
|
hub_always_push=False,
|
|||
|
|
hub_model_id=W-61/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64,
|
|||
|
|
hub_model_revision=main,
|
|||
|
|
hub_private_repo=None,
|
|||
|
|
hub_strategy=HubStrategy.EVERY_SAVE,
|
|||
|
|
hub_token=<HUB_TOKEN>,
|
|||
|
|
ignore_data_skip=False,
|
|||
|
|
include_for_metrics=[],
|
|||
|
|
include_inputs_for_metrics=False,
|
|||
|
|
include_num_input_tokens_seen=False,
|
|||
|
|
include_tokens_per_second=False,
|
|||
|
|
is_encoder_decoder=None,
|
|||
|
|
jit_mode_eval=False,
|
|||
|
|
label_names=None,
|
|||
|
|
label_pad_token_id=-100,
|
|||
|
|
label_smoothing=0.0,
|
|||
|
|
label_smoothing_factor=0.0,
|
|||
|
|
learning_rate=5e-07,
|
|||
|
|
length_column_name=length,
|
|||
|
|
load_best_model_at_end=False,
|
|||
|
|
local_rank=0,
|
|||
|
|
log_level=info,
|
|||
|
|
log_level_replica=warning,
|
|||
|
|
log_on_each_node=True,
|
|||
|
|
logging_dir=outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64/runs/Apr18_09-13-06_d4053,
|
|||
|
|
logging_first_step=True,
|
|||
|
|
logging_nan_inf_filter=True,
|
|||
|
|
logging_steps=1,
|
|||
|
|
logging_strategy=IntervalStrategy.STEPS,
|
|||
|
|
loss_type=sigmoid,
|
|||
|
|
lr_scheduler_kwargs={},
|
|||
|
|
lr_scheduler_type=SchedulerType.COSINE,
|
|||
|
|
max_grad_norm=1.0,
|
|||
|
|
max_length=512,
|
|||
|
|
max_prompt_length=256,
|
|||
|
|
max_steps=-1,
|
|||
|
|
max_target_length=None,
|
|||
|
|
metric_for_best_model=None,
|
|||
|
|
model_adapter_name=None,
|
|||
|
|
model_init_kwargs=None,
|
|||
|
|
mp_parameters=,
|
|||
|
|
neftune_noise_alpha=None,
|
|||
|
|
no_cuda=False,
|
|||
|
|
non_finite_logits_handling=error,
|
|||
|
|
num_train_epochs=1,
|
|||
|
|
optim=OptimizerNames.ADAMW_TORCH,
|
|||
|
|
optim_args=None,
|
|||
|
|
optim_target_modules=None,
|
|||
|
|
output_dir=/scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332,
|
|||
|
|
overwrite_output_dir=False,
|
|||
|
|
padding_value=None,
|
|||
|
|
past_index=-1,
|
|||
|
|
per_device_eval_batch_size=8,
|
|||
|
|
per_device_train_batch_size=8,
|
|||
|
|
post_tokenization_log_dir=None,
|
|||
|
|
post_tokenization_log_samples=0,
|
|||
|
|
precompute_ref_batch_size=None,
|
|||
|
|
precompute_ref_eval_batch_size=None,
|
|||
|
|
precompute_ref_log_probs=False,
|
|||
|
|
prediction_loss_only=False,
|
|||
|
|
push_to_hub=False,
|
|||
|
|
push_to_hub_model_id=None,
|
|||
|
|
push_to_hub_organization=None,
|
|||
|
|
push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
|
|||
|
|
ray_scope=last,
|
|||
|
|
ref_adapter_name=None,
|
|||
|
|
ref_model_init_kwargs=None,
|
|||
|
|
ref_model_mixup_alpha=0.9,
|
|||
|
|
ref_model_sync_steps=64,
|
|||
|
|
reference_free=False,
|
|||
|
|
remove_unused_columns=False,
|
|||
|
|
report_to=['wandb'],
|
|||
|
|
restore_callback_states_from_checkpoint=False,
|
|||
|
|
resume_from_checkpoint=None,
|
|||
|
|
reuse_tokenized_dataset=True,
|
|||
|
|
rpo_alpha=None,
|
|||
|
|
run_name=mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332,
|
|||
|
|
save_on_each_node=False,
|
|||
|
|
save_only_model=False,
|
|||
|
|
save_safetensors=True,
|
|||
|
|
save_steps=200,
|
|||
|
|
save_strategy=SaveStrategy.STEPS,
|
|||
|
|
save_total_limit=2,
|
|||
|
|
seed=42,
|
|||
|
|
sft_weight=0.0,
|
|||
|
|
skip_memory_metrics=True,
|
|||
|
|
sync_ref_model=False,
|
|||
|
|
tf32=None,
|
|||
|
|
tokenization_batch_size=128,
|
|||
|
|
tokenization_mode=online,
|
|||
|
|
tokenized_dataset_cache_dir=/scratch/feng.yulu/dynamic-dpo-v4/tokenized_preferences,
|
|||
|
|
torch_compile=False,
|
|||
|
|
torch_compile_backend=None,
|
|||
|
|
torch_compile_mode=None,
|
|||
|
|
torch_empty_cache_steps=None,
|
|||
|
|
torchdynamo=None,
|
|||
|
|
tp_size=0,
|
|||
|
|
tpu_metrics_debug=False,
|
|||
|
|
tpu_num_cores=None,
|
|||
|
|
trainer_type=epsilon_dpo,
|
|||
|
|
truncation_mode=keep_end,
|
|||
|
|
use_cpu=False,
|
|||
|
|
use_ipex=False,
|
|||
|
|
use_legacy_prediction_loop=False,
|
|||
|
|
use_liger_kernel=False,
|
|||
|
|
use_mps_device=False,
|
|||
|
|
wandb_project=ood-run-4xh200,
|
|||
|
|
warmup_ratio=0.1,
|
|||
|
|
warmup_steps=0,
|
|||
|
|
weight_decay=0.0,
|
|||
|
|
)
|
|||
|
|
2026-04-18 09:13:06 - INFO - __main__ - W&B project: ood-run-4xh200
|
|||
|
|
2026-04-18 09:13:06 - INFO - __main__ - Epsilon-DPO parameters: beta=0.1, epsilon=0.01, gradient_accumulation_steps=2
|
|||
|
|
2026-04-18 09:13:06 - INFO - __main__ - Using persistent HF datasets cache at /scratch/feng.yulu/dynamic-dpo-v4/hf/datasets
|
|||
|
|
Normalizing raw HH preferences (train): 0%| | 0/43598 [00:00<?, ? examples/s]2026-04-18 09:13:09 - WARNING - __main__ - Dropped 237 non-canonical HH preference examples from split `train` before normalization (126 x HH preprocessing expects exactly one final assistant response in chosen/rejected suffixes., 111 x HH chosen/rejected transcripts must each contain a divergent assistant response.).
|
|||
|
|
Normalizing raw HH preferences (train): 0%| | 0/43598 [00:00<?, ? examples/s]
Normalizing raw HH preferences (train): 2%|█▍ | 1042/43598 [00:00<00:04, 10368.07 examples/s]
Normalizing raw HH preferences (train): 0%| | 0/43598 [00:00<?, ? examples/s]
Normalizing raw HH preferences (train): 3%|█▌ | 1129/43598 [00:00<00:03, 11234.30 examples/s]
Normalizing raw HH preferences (train): 5%|███▎ | 2342/43598 [00:00<00:03, 11907.62 examples/s]
Normalizing raw HH preferences (train): 2%|█▍ | 1047/43598 [00:00<00:04, 10427.65 examples/s]
Normalizing raw HH preferences (train): 0%| | 0/43598 [00:00<?, ? examples/s]
Normalizing raw HH preferences (train): 6%|███▍ | 2429/43598 [00:00<00:03, 12268.86 examples/s]
Normalizing raw HH preferences (train): 9%|█████▏ | 3706/43598 [00:00<00:03, 12396.17 examples/s]
Normalizing raw HH preferences (train): 5%|███▏ | 2300/43598 [00:00<00:03, 11657.68 examples/s]
Normalizing raw HH preferences (train): 2%|█▍ | 1049/43598 [00:00<00:04, 10430.67 examples/s]
Normalizing raw HH preferences (train): 9%|█████▏ | 3716/43598 [00:00<00:03, 12539.13 examples/s]
Normalizing raw HH preferences (train): 11%|██████▉ | 4974/43598 [00:00<00:03, 12501.81 examples/s]
Normalizing raw HH preferences (train): 8%|████▉ | 3551/43598 [00:00<00:03, 12038.88 examples/s]
Normalizing raw HH preferences (train): 5%|███▏ | 2297/43598 [00:00<00:03, 11629.56 examples/s]
Normalizing raw HH preferences (train): 11%|██████▉ | 4980/43598 [00:00<00:03, 12575.02 examples/s]
Normalizing raw HH preferences (train): 11%|██████▋ | 4782/43598 [00:00<00:03, 12145.08 examples/s]
Normalizing raw HH preferences (train): 8%|████▉ | 3531/43598 [00:00<00:03, 11951.25 examples/s]
Normalizing raw HH preferences (train): 11%|██████▋ | 4768/43598 [00:00<00:03, 12115.36 examples/s]
Normalizing raw HH preferences (train): 15%|█████████ | 6344/43598 [00:00<00:04, 8272.67 examples/s]
Normalizing raw HH preferences (train): 15%|█████████ | 6335/43598 [00:00<00:04, 8253.67 examples/s]
Normalizing raw HH preferences (train): 18%|██████████▉ | 7703/43598 [00:00<00:03, 9438.99 examples/s]
Normalizing raw HH preferences (train): 14%|████████▉ | 6278/43598 [00:00<00:04, 8019.30 examples/s]
Normalizing raw HH preferences (train): 18%|██████████▉ | 7698/43598 [00:00<00:03, 9424.33 examples/s]
Normalizing raw HH preferences (train): 21%|████████████▌ | 8978/43598 [00:00<
|
|||
|
|
Normalizing raw HH preferences (train): 100%|████████████████████████████████████████████████████████████▉| 43534/43598 [00:03<00:00, 8805.15 examples/s]
Normalizing raw HH preferences (train): 100%|████████████████████████████████████████████████████████████▉| 43539/43598 [00:03<00:00, 9287.89 examples/s]
Normalizing raw HH preferences (train): 100%|████████████████████████████████████████████████████████████| 43598/43598 [00:04<00:00, 10612.00 examples/s]
|
|||
|
|
Normalizing raw HH preferences (train): 100%|████████████████████████████████████████████████████████████| 43598/43598 [00:04<00:00, 10529.41 examples/s]
|
|||
|
|
Normalizing raw HH preferences (train): 100%|████████████████████████████████████████████████████████████| 43598/43598 [00:04<00:00, 10514.13 examples/s]
|
|||
|
|
Normalizing raw HH preferences (test): 0%| | 0/2339 [00:00<?, ? examples/s]
Normalizing raw HH preferences (test): 43%|███████████████████████████▍ | 1001/2339 [00:00<00:00, 9964.28 examples/s]2026-04-18 09:13:13 - WARNING - __main__ - Dropped 15 non-canonical HH preference examples from split `test` before normalization (9 x HH preprocessing expects exactly one final assistant response in chosen/rejected suffixes., 6 x HH chosen/rejected transcripts must each contain a divergent assistant response.).
|
|||
|
|
Normalizing raw HH preferences (test): 0%| | 0/2339 [00:00<?, ? examples/s]
Normalizing raw HH preferences (test): 97%|████████████████████████████████████████████████████████████▊ | 2258/2339 [00:00<00:00, 11488.85 examples/s]
Normalizing raw HH preferences (test): 100%|███████████████████████████████████████████████████████████████| 2339/2339 [00:00<00:00, 10024.78 examples/s]
|
|||
|
|
Normalizing raw HH preferences (test): 0%| | 0/2339 [00:00<?, ? examples/s]
Normalizing raw HH preferences (test): 46%|█████████████████████████████▏ | 1083/2339 [00:00<00:00, 10781.12 examples/s]
Normalizing raw HH preferences (test): 46%|████████████████████████████▋ | 1066/2339 [00:00<00:00, 10610.58 examples/s]
Normalizing raw HH preferences (test): 100%|██████████████████████████████████████████████████████████████▊| 2331/2339 [00:00<00:00, 11775.52 examples/s]
Normalizing raw HH preferences (test): 0%| | 0/2339 [00:00<?, ? examples/s]
Normalizing raw HH preferences (test): 100%|████████████████████████████████████████████████████████████████| 2339/2339 [00:00<00:00, 9446.81 examples/s]
|
|||
|
|
2026-04-18 09:13:14 - INFO - __main__ - Training on the following splits: ['train : 43598', 'test : 2339']
|
|||
|
|
[INFO|tokenization_utils_base.py:2058] 2026-04-18 09:13:14,127 >> loading file tokenizer.model
|
|||
|
|
[INFO|tokenization_utils_base.py:2058] 2026-04-18 09:13:14,127 >> loading file tokenizer.json
|
|||
|
|
[INFO|tokenization_utils_base.py:2058] 2026-04-18 09:13:14,127 >> loading file added_tokens.json
|
|||
|
|
[INFO|tokenization_utils_base.py:2058] 2026-04-18 09:13:14,127 >> loading file special_tokens_map.json
|
|||
|
|
[INFO|tokenization_utils_base.py:2058] 2026-04-18 09:13:14,127 >> loading file tokenizer_config.json
|
|||
|
|
[INFO|tokenization_utils_base.py:2058] 2026-04-18 09:13:14,127 >> loading file chat_template.jinja
|
|||
|
|
Normalizing raw HH preferences (test): 98%|█████████████████████████████████████████████████████████████▋ | 2289/2339 [00:00<00:00, 11556.98 examples/s]
Normalizing raw HH preferences (test): 43%|███████████████████████████▎ | 1000/2339 [00:00<00:00, 9566.10 examples/s]
Normalizing raw HH preferences (test): 100%|████████████████████████████████████████████████████████████████| 2339/2339 [00:00<00:00, 9002.76 examples/s]
|
|||
|
|
Normalizing raw HH preferences (test): 95%|███████████████████████████████████████████████████████████▊ | 2220/2339 [00:00<00:00, 11086.23 examples/s]
Normalizing raw HH preferences (test): 100%|████████████████████████████████████████████████████████████████| 2339/2339 [00:00<00:00, 8890.42 examples/s]
|
|||
|
|
/home/feng.yulu/dynamic-dpo-v4/scripts/tokenized_dpo_trainer.py:391: UserWarning: You passed a model_id to the trainer. This will automatically create an `AutoModelForCausalLM` or a `PeftModel` (if you passed a `peft_config`) for you.
|
|||
|
|
warnings.warn(
|
|||
|
|
2026-04-18 09:13:14 - INFO - __main__ - Processed train sample 41905:
|
|||
|
|
|
|||
|
|
Prompt:
|
|||
|
|
<|system|>
|
|||
|
|
</s>
|
|||
|
|
<|user|>
|
|||
|
|
What is the best way to clean my refrigerator?</s>
|
|||
|
|
<|assistant|>
|
|||
|
|
Well, I have a few ideas, but the best way to know what works best is to run some tests and then analyze the data to figure out which technique works best.</s>
|
|||
|
|
<|user|>
|
|||
|
|
What are your ideas?</s>
|
|||
|
|
|
|||
|
|
|
|||
|
|
Chosen:
|
|||
|
|
<|assistant|>
|
|||
|
|
Some of my ideas include wiping down the interior with a rag, using bleach and water to wipe down the door, emptying the contents of the fridge to scrub the shelves with a clean cloth, cleaning the interior of the fridge with vinegar and water, and using a vacuum cleaner to clean the interior.</s>
|
|||
|
|
|
|||
|
|
|
|||
|
|
Rejected:
|
|||
|
|
<|assistant|>
|
|||
|
|
I was thinking that you could try different types of soap, which could be a variation of “soap,” like antibacterial soap. I was also thinking that you could try using different tools, like a toothbrush, to clean the inside. Or I was thinking that you could try different ways to get the gunk out, like using a squeegee, or using a paper towel. I was also thinking you could try other types of cleaning, like vacuuming, but I think that could have the opposite of the desired effect.</s>
|
|||
|
|
|
|||
|
|
/home/feng.yulu/dynamic-dpo-v4/scripts/tokenized_dpo_trainer.py:391: UserWarning: You passed a model_id to the trainer. This will automatically create an `AutoModelForCausalLM` or a `PeftModel` (if you passed a `peft_config`) for you.
|
|||
|
|
warnings.warn(
|
|||
|
|
[INFO|configuration_utils.py:691] 2026-04-18 09:13:14,482 >> loading configuration file /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-sft-hh-helpful-4xh200-batch-64-20260418-015332/config.json
|
|||
|
|
[INFO|configuration_utils.py:765] 2026-04-18 09:13:14,487 >> Model config MistralConfig {
|
|||
|
|
"architectures": [
|
|||
|
|
"MistralForCausalLM"
|
|||
|
|
],
|
|||
|
|
"attention_dropout": 0.0,
|
|||
|
|
"bos_token_id": 1,
|
|||
|
|
"eos_token_id": 2,
|
|||
|
|
"head_dim": 128,
|
|||
|
|
"hidden_act": "silu",
|
|||
|
|
"hidden_size": 4096,
|
|||
|
|
"initializer_range": 0.02,
|
|||
|
|
"intermediate_size": 14336,
|
|||
|
|
"max_position_embeddings": 32768,
|
|||
|
|
"model_type": "mistral",
|
|||
|
|
"num_attention_heads": 32,
|
|||
|
|
"num_hidden_layers": 32,
|
|||
|
|
"num_key_value_heads": 8,
|
|||
|
|
"rms_norm_eps": 1e-05,
|
|||
|
|
"rope_theta": 1000000.0,
|
|||
|
|
"sliding_window": null,
|
|||
|
|
"tie_word_embeddings": false,
|
|||
|
|
"torch_dtype": "bfloat16",
|
|||
|
|
"transformers_version": "4.51.0",
|
|||
|
|
"use_cache": false,
|
|||
|
|
"vocab_size": 32768
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/home/feng.yulu/dynamic-dpo-v4/scripts/tokenized_dpo_trainer.py:391: UserWarning: You passed a model_id to the trainer. This will automatically create an `AutoModelForCausalLM` or a `PeftModel` (if you passed a `peft_config`) for you.
|
|||
|
|
warnings.warn(
|
|||
|
|
/home/feng.yulu/dynamic-dpo-v4/scripts/tokenized_dpo_trainer.py:391: UserWarning: You passed a model_id to the trainer. This will automatically create an `AutoModelForCausalLM` or a `PeftModel` (if you passed a `peft_config`) for you.
|
|||
|
|
warnings.warn(
|
|||
|
|
[INFO|modeling_utils.py:1121] 2026-04-18 09:13:14,750 >> loading weights file /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-sft-hh-helpful-4xh200-batch-64-20260418-015332/model.safetensors.index.json
|
|||
|
|
[INFO|modeling_utils.py:2167] 2026-04-18 09:13:14,753 >> Instantiating MistralForCausalLM model under default dtype torch.bfloat16.
|
|||
|
|
[WARNING|logging.py:328] 2026-04-18 09:13:14,755 >> You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
|
|||
|
|
[INFO|configuration_utils.py:1142] 2026-04-18 09:13:14,756 >> Generate config GenerationConfig {
|
|||
|
|
"bos_token_id": 1,
|
|||
|
|
"eos_token_id": 2,
|
|||
|
|
"use_cache": false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
[WARNING|logging.py:328] 2026-04-18 09:13:14,756 >> You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
|
|||
|
|
[WARNING|logging.py:328] 2026-04-18 09:13:14,760 >> You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
|
|||
|
|
Loading checkpoint shards: 0%| | 0/6 [00:00<?, ?it/s]
Loading checkpoint shards: 0%| | 0/6 [00:00<?, ?it/s]
Loading checkpoint shards: 0%| | 0/6 [00:00<?, ?it/s]
Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 103.84it/s]
|
|||
|
|
Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 88.34it/s]
|
|||
|
|
Loading checkpoint shards: 0%| | 0/6 [00:00<?, ?it/s]
Loading checkpoint shards: 0%| | 0/6 [00:00<?, ?it/s]
Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 167.85it/s]
|
|||
|
|
[WARNING|logging.py:328] 2026-04-18 09:13:14,952 >> You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
|
|||
|
|
Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 152.88it/s]
|
|||
|
|
[WARNING|trainer.py:821] 2026-04-18 09:13:14,955 >> Trainer.tokenizer is now deprecated. You should use `Trainer.processing_class = processing_class` instead.
|
|||
|
|
[WARNING|trainer.py:821] 2026-04-18 09:13:14,960 >> Trainer.tokenizer is now deprecated. You should use `Trainer.processing_class = processing_class` instead.
|
|||
|
|
Loading checkpoint shards: 0%| | 0/6 [00:00<?, ?it/s]
Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 190.94it/s]
|
|||
|
|
Loading checkpoint shards: 0%| | 0/6 [00:00<?, ?it/s]
Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:00<00:00, 148.67it/s]
|
|||
|
|
[WARNING|trainer.py:821] 2026-04-18 09:13:15,135 >> Trainer.tokenizer is now deprecated. You should use `Trainer.processing_class = processing_class` instead.
|
|||
|
|
Loading checkpoint shards: 17%|███████████████▏ | 1/6 [00:10<00:52, 10.52s/it]
Loading checkpoint shards: 33%|██████████████████████████████▎ | 2/6 [00:12<00:21, 5.46s/it]
Loading checkpoint shards: 50%|█████████████████████████████████████████████▌ | 3/6 [00:14<00:11, 3.95s/it]
Loading checkpoint shards: 67%|████████████████████████████████████████████████████████████▋ | 4/6 [00:16<00:06, 3.21s/it]
Loading checkpoint shards: 83%|███████████████████████████████████████████████████████████████████████████▊ | 5/6 [00:18<00:02, 2.78s/it]
Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:20<00:00, 2.44s/it]
Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:20<00:00, 3.41s/it]
|
|||
|
|
[INFO|modeling_utils.py:4926] 2026-04-18 09:13:35,258 >> All model checkpoint weights were used when initializing MistralForCausalLM.
|
|||
|
|
|
|||
|
|
[INFO|modeling_utils.py:4934] 2026-04-18 09:13:35,258 >> All the weights of MistralForCausalLM were initialized from the model checkpoint at /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-sft-hh-helpful-4xh200-batch-64-20260418-015332.
|
|||
|
|
If your task is similar to the task the model of the checkpoint was trained on, you can already use MistralForCausalLM for predictions without further training.
|
|||
|
|
[INFO|configuration_utils.py:1095] 2026-04-18 09:13:35,260 >> loading configuration file /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-sft-hh-helpful-4xh200-batch-64-20260418-015332/generation_config.json
|
|||
|
|
[INFO|configuration_utils.py:1142] 2026-04-18 09:13:35,260 >> Generate config GenerationConfig {
|
|||
|
|
"bos_token_id": 1,
|
|||
|
|
"eos_token_id": 2
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
[INFO|configuration_utils.py:691] 2026-04-18 09:13:35,262 >> loading configuration file /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-sft-hh-helpful-4xh200-batch-64-20260418-015332/config.json
|
|||
|
|
[INFO|configuration_utils.py:765] 2026-04-18 09:13:35,262 >> Model config MistralConfig {
|
|||
|
|
"architectures": [
|
|||
|
|
"MistralForCausalLM"
|
|||
|
|
],
|
|||
|
|
"attention_dropout": 0.0,
|
|||
|
|
"bos_token_id": 1,
|
|||
|
|
"eos_token_id": 2,
|
|||
|
|
"head_dim": 128,
|
|||
|
|
"hidden_act": "silu",
|
|||
|
|
"hidden_size": 4096,
|
|||
|
|
"initializer_range": 0.02,
|
|||
|
|
"intermediate_size": 14336,
|
|||
|
|
"max_position_embeddings": 32768,
|
|||
|
|
"model_type": "mistral",
|
|||
|
|
"num_attention_heads": 32,
|
|||
|
|
"num_hidden_layers": 32,
|
|||
|
|
"num_key_value_heads": 8,
|
|||
|
|
"rms_norm_eps": 1e-05,
|
|||
|
|
"rope_theta": 1000000.0,
|
|||
|
|
"sliding_window": null,
|
|||
|
|
"tie_word_embeddings": false,
|
|||
|
|
"torch_dtype": "bfloat16",
|
|||
|
|
"transformers_version": "4.51.0",
|
|||
|
|
"use_cache": false,
|
|||
|
|
"vocab_size": 32768
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
[INFO|modeling_utils.py:1121] 2026-04-18 09:13:35,264 >> loading weights file /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-sft-hh-helpful-4xh200-batch-64-20260418-015332/model.safetensors.index.json
|
|||
|
|
[INFO|modeling_utils.py:2167] 2026-04-18 09:13:35,264 >> Instantiating MistralForCausalLM model under default dtype torch.bfloat16.
|
|||
|
|
[INFO|configuration_utils.py:1142] 2026-04-18 09:13:35,267 >> Generate config GenerationConfig {
|
|||
|
|
"bos_token_id": 1,
|
|||
|
|
"eos_token_id": 2,
|
|||
|
|
"use_cache": false
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
Loading checkpoint shards: 0%| | 0/6 [00:00<?, ?it/s]
Loading checkpoint shards: 17%|███████████████▏ | 1/6 [00:02<00:11, 2.27s/it]
Loading checkpoint shards: 33%|██████████████████████████████▎ | 2/6 [00:04<00:08, 2.20s/it]
Loading checkpoint shards: 50%|█████████████████████████████████████████████▌ | 3/6 [00:06<00:06, 2.13s/it]
Loading checkpoint shards: 67%|████████████████████████████████████████████████████████████▋ | 4/6 [00:08<00:04, 2.10s/it]
Loading checkpoint shards: 83%|███████████████████████████████████████████████████████████████████████████▊ | 5/6 [00:10<00:02, 2.08s/it]
Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:12<00:00, 1.97s/it]
Loading checkpoint shards: 100%|███████████████████████████████████████████████████████████████████████████████████████████| 6/6 [00:12<00:00, 2.05s/it]
|
|||
|
|
[INFO|modeling_utils.py:4926] 2026-04-18 09:13:47,615 >> All model checkpoint weights were used when initializing MistralForCausalLM.
|
|||
|
|
|
|||
|
|
[INFO|modeling_utils.py:4934] 2026-04-18 09:13:47,615 >> All the weights of MistralForCausalLM were initialized from the model checkpoint at /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-sft-hh-helpful-4xh200-batch-64-20260418-015332.
|
|||
|
|
If your task is similar to the task the model of the checkpoint was trained on, you can already use MistralForCausalLM for predictions without further training.
|
|||
|
|
[INFO|configuration_utils.py:1095] 2026-04-18 09:13:47,617 >> loading configuration file /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-sft-hh-helpful-4xh200-batch-64-20260418-015332/generation_config.json
|
|||
|
|
[INFO|configuration_utils.py:1142] 2026-04-18 09:13:47,617 >> Generate config GenerationConfig {
|
|||
|
|
"bos_token_id": 1,
|
|||
|
|
"eos_token_id": 2
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
[WARNING|trainer.py:821] 2026-04-18 09:13:47,618 >> Trainer.tokenizer is now deprecated. You should use `Trainer.processing_class = processing_class` instead.
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:13:47,623 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
Tokenizing train (num_proc=12): 0%| | 0/43598 [00:00<?, ? examples/s]
Tokenizing train (num_proc=12): 0%|▏ | 128/43598 [01:00<5:43:40, 2.11 examples/s]
Tokenizing train (num_proc=12): 1%|▋ | 384/43598 [01:00<1:28:56, 8.10 examples/s]
Tokenizing train (num_proc=12): 1%|█ | 640/43598 [01:01<43:14, 16.56 examples/s]
Tokenizing train (num_proc=12): 2%|█▌ | 896/43598 [01:01<25:00, 28.47 examples/s]
Tokenizing train (num_proc=12): 3%|█▉ | 1152/43598 [01:01<15:40, 45.14 examples/s]
Tokenizing train (num_proc=12): 3%|██▎ | 1408/43598 [01:01<10:18, 68.27 examples/s]
Tokenizing train (num_proc=12): 4%|██▋ | 1664/43598 [01:01<06:59, 100.07 examples/s]
Tokenizing train (num_proc=12): 4%|███▏ | 1920/43598 [01:02<04:52, 142.54 examples/s]
Tokenizing train (num_proc=12): 5%|███▌ | 2176/43598 [01:02<03:28, 198.21 examples/s]
Tokenizing train (num_proc=12): 6%|███▉ | 2432/43598 [01:02<02:32, 269.17 examples/s]
Tokenizing train (num_proc=12): 6%|████▍ | 2688/43598 [01:02<01:54, 356.04 examples/s]
Tokenizing train (num_proc=12): 7%|████▊ | 2944/43598 [01:02<01:28, 459.31 examples/s]
Tokenizing train (num_proc=12): 7%|█████▏ | 3200/43598 [01:02<01:11, 567.11 examples/s]
Tokenizing train (num_proc=12): 8%|█████▋ | 3456/43598 [01:03<00:58, 681.99 examples/s]
Tokenizing train (num_proc=12): 8%|█████▉ | 3634/43598 [01:03<01:02, 643.72 examples/s]
Tokenizing train (num_proc=12): 8%|█████▉ | 3634/43598 [01:18<01:02, 643.72 examples/s]
Tokenizing train (num_proc=12): 9%|██████▏ | 3762/43598 [01:39<36:31, 18.18 examples/s]
Tokenizing train (num_proc=12): 9%|██████▋ | 4018/43598 [01:39<23:34, 27.99 examples/s]
Tokenizing train (num_proc=12): 10%|███████ | 4274/43598 [01:39<15:43, 41.69 examples/s]
Tokenizing train (num_proc=12): 10%|███████▍ | 4530/43598 [01:39<10:42, 60.83 examples/s]
Tokenizing train (num_proc=12): 11%|███████▉ | 4786/43598 [01:40<07:25, 87.13 examples/s]
Tokenizing train (num_proc=12): 12%|████████▏ | 5042/43598 [01:40<05:14, 122.77 examples/s]
Tokenizing train (num_proc=12): 12%|████████▋ | 5298/43598 [01:40<03:44, 170.23 examples/s]
Tokenizing train (num_proc=12): 13%|█████████ | 5554/43598 [01:40<02:43, 233.29 examples/s
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:22:40,439 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
Saving the dataset (0/2 shards): 0%| | 0/43598 [00:00<?, ? examples/s]
Saving the dataset (0/2 shards): 9%|██████▏ | 4000/43598 [00:00<00:01, 24898.99 examples/s]
Saving the dataset (0/2 shards): 16%|██████████▉ | 7000/43598 [00:00<00:01, 20041.45 examples/s]
Saving the dataset (0/2 shards): 23%|███████████████▎ | 10000/43598 [00:00<00:01, 19687.83 examples/s]
Saving the dataset (0/2 shards): 28%|██████████████████▍ | 12000/43598 [00:00<00:01, 17520.07 examples/s]
Saving the dataset (0/2 shards): 37%|████████████████████████▌ | 16000/43598 [00:00<00:01, 18279.51 examples/s]
Saving the dataset (0/2 shards): 41%|███████████████████████████▋ | 18000/43598 [00:01<00:01, 16149.99 examples/s]
Saving the dataset (0/2 shards): 50%|█████████████████████████████████▌ | 21799/43598 [00:01<00:01, 18986.38 examples/s]
Saving the dataset (1/2 shards): 50%|█████████████████████████████████▌ | 21799/43598 [00:01<00:01, 18986.38 examples/s]
Saving the dataset (1/2 shards): 57%|██████████████████████████████████████▋ | 24799/43598 [00:01<00:01, 9632.90 examples/s]
Saving the dataset (1/2 shards): 61%|█████████████████████████████████████████▏ | 26799/43598 [00:01<00:01, 10637.06 examples/s]
Saving the dataset (1/2 shards): 66%|████████████████████████████████████████████▎ | 28799/43598 [00:02<00:01, 11664.08 examples/s]
Saving the dataset (1/2 shards): 75%|██████████████████████████████████████████████████▍ | 32799/43598 [00:02<00:00, 14804.80 examples/s]
Saving the dataset (1/2 shards): 80%|█████████████████████████████████████████████████████▍ | 34799/43598 [00:02<00:00, 15145.78 examples/s]
Saving the dataset (1/2 shards): 87%|██████████████████████████████████████████████████████████ | 37799/43598 [00:02<00:00, 17343.60 examples/s]
Saving the dataset (1/2 shards): 96%|████████████████████████████████████████████████████████████████▏ | 41799/43598 [00:02<00:00, 19903.85 examples/s]
Saving the dataset (2/2 shards): 100%|███████████████████████████████████████████████████████████████████| 43598/43598 [00:03<00:00, 19903.85 examples/s]
Saving the dataset (2/2 shards): 100%|███████████████████████████████████████████████████████████████████| 43598/43598 [00:03<00:00, 13307.19 examples/s
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:22:45,913 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
Tokenizing test (num_proc=12): 0%| | 0/2339 [00:00<?, ? examples/s]
Tokenizing test (num_proc=12): 5%|████ | 128/2339 [00:43<12:32, 2.94 examples/s]
Tokenizing test (num_proc=12): 8%|██████▎ | 195/2339 [00:43<07:00, 5.10 examples/s]
Tokenizing test (num_proc=12): 8%|██████▎ | 195/2339 [01:01<07:00, 5.10 examples/s]
Tokenizing test (num_proc=12): 14%|██████████▎ | 323/2339 [01:17<07:46, 4.33 examples/s]
Tokenizing test (num_proc=12): 22%|████████████████▌ | 518/2339 [01:49<05:56, 5.12 examples/s]
Tokenizing test (num_proc=12): 30%|██████████████████████▊ | 713/2339 [02:20<04:53, 5.54 examples/s]
Tokenizing test (num_proc=12): 39%|█████████████████████████████ | 908/2339 [02:51<04:05, 5.84 examples/s]
Tokenizing test (num_proc=12): 42%|███████████████████████████████▎ | 975/2339 [02:51<03:17, 6.92 examples/s]
Tokenizing test (num_proc=12): 42%|███████████████████████████████▎ | 975/2339 [03:01<03:17, 6.92 examples/s]
Tokenizing test (num_proc=12): 47%|██████████████████████████████████▉ | 1103/2339 [03:25<03:44, 5.51 examples/s]
Tokenizing test (num_proc=12): 55%|█████████████████████████████████████████ | 1298/2339 [04:01<03:09, 5.49 examples/s]
Tokenizing test (num_proc=12): 64%|███████████████████████████████████████████████▏ | 1493/2339 [04:39<02:37, 5.37 examples/s]
Tokenizing test (num_proc=12): 72%|█████████████████████████████████████████████████████▍ | 1688/2339 [05:16<02:02, 5.33 examples/s]
Tokenizing test (num_proc=12): 75%|███████████████████████████████████████████████████████▌ | 1755/2339 [05:16<01:34, 6.21 examples/s]
Tokenizing test (num_proc=12): 75%|███████████████████████████████████████████████████████▌ | 1755/2339 [05:31<01:34, 6.21 examples/s]
Tokenizing test (num_proc=12): 81%|███████████████████████████████████████████████████████████▌ | 1883/2339 [05:54<01:30, 5.02 examples/s]
Tokenizing test (num_proc=12): 89%|█████████████████████████████████████████████████████████████████▋ | 2078/2339 [06:27<00:49, 5.29 examples/s]
Tokenizing test (num_proc=12): 97%|███████████████████████████████████████████<E29688>
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:00,693 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
Saving the dataset (0/1 shards): 0%| | 0/2339 [00:00<?, ? examples/s]
Saving the dataset (0/1 shards): 100%|█████████████████████████████████████████████████████████████████████| 2339/2339 [00:00<00:00, 22183.00 examples/s]
Saving the dataset (1/1 shards): 100%|█████████████████████████████████████████████████████████████████████| 2339/2339 [00:00<00:00, 22183.00 examples/s]
Saving the dataset (1/1 shards): 100%|█████████████████████████████████████████████████████████████████████| 2339/2339 [00:00<00:00, 15546.92 examples/s]
|
|||
|
|
/home/feng.yulu/dynamic-dpo-v4/scripts/tokenized_dpo_trainer.py:521: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `EpsilonDPOTrainer.__init__`. Use `processing_class` instead.
|
|||
|
|
super().__init__(
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:02,080 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:02,080 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:02,081 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:02,239 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:02,239 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:02,240 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:02,240 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:02,241 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:02,241 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:02,263 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:02,263 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
[WARNING|trainer.py:816] 2026-04-18 09:31:02,263 >> Trainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.
|
|||
|
|
/home/feng.yulu/dynamic-dpo-v4/scripts/tokenized_dpo_trainer.py:521: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `EpsilonDPOTrainer.__init__`. Use `processing_class` instead.
|
|||
|
|
super().__init__(
|
|||
|
|
/home/feng.yulu/dynamic-dpo-v4/scripts/tokenized_dpo_trainer.py:521: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `EpsilonDPOTrainer.__init__`. Use `processing_class` instead.
|
|||
|
|
super().__init__(
|
|||
|
|
/home/feng.yulu/dynamic-dpo-v4/scripts/tokenized_dpo_trainer.py:521: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `EpsilonDPOTrainer.__init__`. Use `processing_class` instead.
|
|||
|
|
super().__init__(
|
|||
|
|
[INFO|trainer.py:748] 2026-04-18 09:31:02,395 >> Using auto half precision backend
|
|||
|
|
/home/feng.yulu/.conda/envs/dpo_venv/lib/python3.11/site-packages/accelerate/accelerator.py:1557: UserWarning: Upcasted low precision parameters in MistralForCausalLM because mixed precision turned on in FSDP. Affects: model.embed_tokens.weight, model.norm.weight, lm_head.weight.
|
|||
|
|
warnings.warn(
|
|||
|
|
/home/feng.yulu/.conda/envs/dpo_venv/lib/python3.11/site-packages/accelerate/accelerator.py:1557: UserWarning: Upcasted low precision parameters in MistralDecoderLayer because mixed precision turned on in FSDP. Affects: self_attn.q_proj.weight, self_attn.k_proj.weight, self_attn.v_proj.weight, self_attn.o_proj.weight, mlp.gate_proj.weight, mlp.up_proj.weight, mlp.down_proj.weight, input_layernorm.weight, post_attention_layernorm.weight.
|
|||
|
|
warnings.warn(
|
|||
|
|
/home/feng.yulu/.conda/envs/dpo_venv/lib/python3.11/site-packages/accelerate/accelerator.py:1563: UserWarning: FSDP upcast of low precision parameters may affect the precision of model checkpoints.
|
|||
|
|
warnings.warn(
|
|||
|
|
[INFO|trainer.py:2414] 2026-04-18 09:31:13,913 >> ***** Running training *****
|
|||
|
|
[INFO|trainer.py:2415] 2026-04-18 09:31:13,913 >> Num examples = 43,598
|
|||
|
|
[INFO|trainer.py:2416] 2026-04-18 09:31:13,913 >> Num Epochs = 1
|
|||
|
|
[INFO|trainer.py:2417] 2026-04-18 09:31:13,913 >> Instantaneous batch size per device = 8
|
|||
|
|
[INFO|trainer.py:2420] 2026-04-18 09:31:13,913 >> Total train batch size (w. parallel, distributed & accumulation) = 64
|
|||
|
|
[INFO|trainer.py:2421] 2026-04-18 09:31:13,913 >> Gradient Accumulation steps = 2
|
|||
|
|
[INFO|trainer.py:2422] 2026-04-18 09:31:13,913 >> Total optimization steps = 681
|
|||
|
|
[INFO|trainer.py:2423] 2026-04-18 09:31:13,914 >> Number of trainable parameters = 1,812,005,888
|
|||
|
|
[INFO|integration_utils.py:831] 2026-04-18 09:31:13,914 >> Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true"
|
|||
|
|
wandb: Currently logged in as: can-not-fand (can-not-fand-northeastern-university). Use `wandb login --relogin` to force relogin
|
|||
|
|
wandb: wandb version 0.26.0 is available! To upgrade, please run:
|
|||
|
|
wandb: $ pip install wandb --upgrade
|
|||
|
|
wandb: Tracking run with wandb version 0.17.5
|
|||
|
|
wandb: Run data is saved locally in /scratch/feng.yulu/dynamic-dpo-v4/wandb/wandb/run-20260418_093116-7dwx2wac
|
|||
|
|
wandb: Run `wandb offline` to turn off syncing.
|
|||
|
|
wandb: Syncing run mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332
|
|||
|
|
wandb: ⭐️ View project at https://wandb.ai/can-not-fand-northeastern-university/ood-run-4xh200
|
|||
|
|
wandb: 🚀 View run at https://wandb.ai/can-not-fand-northeastern-university/ood-run-4xh200/runs/7dwx2wac
|
|||
|
|
0%| | 0/681 [00:00<?, ?it/s][WARNING|modeling_utils.py:1713] 2026-04-18 09:31:23,466 >> Could not estimate the number of tokens of the input, floating-point operations will not be computed
|
|||
|
|
[WARNING|modeling_utils.py:1713] 2026-04-18 09:31:23,467 >> Could not estimate the number of tokens of the input, floating-point operations will not be computed
|
|||
|
|
[WARNING|modeling_utils.py:1713] 2026-04-18 09:31:23,471 >> Could not estimate the number of tokens of the input, floating-point operations will not be computed
|
|||
|
|
[WARNING|modeling_utils.py:1713] 2026-04-18 09:31:23,484 >> Could not estimate the number of tokens of the input, floating-point operations will not be computed
|
|||
|
|
0%|▏ | 1/681 [00:02<30:36, 2.70s/it]
{'loss': 1.3843, 'grad_norm': 153.6117401123047, 'learning_rate': 0.0, 'rewards/chosen': 0.0014821073273196816, 'rewards/rejected': -0.0007488295086659491, 'rewards/accuracies': 0.59375, 'rewards/margins': 0.002230936661362648, 'logps/chosen': -31.484785079956055, 'logps/rejected': -79.35993194580078, 'logps/ref_chosen': -31.500404357910156, 'logps/ref_rejected': -79.35133361816406, 'logits/chosen': -3.435748338699341, 'logits/rejected': -3.461001396179199, 'kl/p_epsilon_steps': 0.609375, 'kl/n_epsilon_steps': 0.390625, 'epsilon_dpo/beta': 0.09979122877120972, 'epsilon_dpo/loss_margin_mean': 0.024216145277023315, 'epsilon_dpo/beta_margin_mean': 0.0022308863699436188, 'epsilon_dpo/beta_margin_std': 0.028449052944779396, 'epsilon_dpo/beta_margin_grad_mean': -0.4994426667690277, 'epsilon_dpo/beta_margin_grad_std': 0.007109890226274729, 'kl/beta': 0.10000000149011612, 'kl/avg_steps': 0.21875, 'epoch': 0.0}
|
|||
|
|
0%|▏ | 1/681 [00:02<30:36, 2.70s/it]
0%|▎ | 2/681 [00:05<29:49, 2.64s/it]
{'loss': 1.3875, 'grad_norm': 155.6954803466797, 'learning_rate': 7.246376811594203e-09, 'rewards/chosen': -8.885323768481612e-05, 'rewards/rejected': 0.0009472252568230033, 'rewards/accuracies': 0.546875, 'rewards/margins': -0.0010360784363001585, 'logps/chosen': -36.63703918457031, 'logps/rejected': -80.44050598144531, 'logps/ref_chosen': -36.63695526123047, 'logps/ref_rejected': -80.44895935058594, 'logits/chosen': -3.517317533493042, 'logits/rejected': -3.4876961708068848, 'kl/p_epsilon_steps': 0.53125, 'kl/n_epsilon_steps': 0.46875, 'epsilon_dpo/beta': 0.09972933679819107, 'epsilon_dpo/loss_margin_mean': -0.008541211485862732, 'epsilon_dpo/beta_margin_mean': -0.0010360678425058722, 'epsilon_dpo/beta_margin_std': 0.024278299883008003, 'epsilon_dpo/beta_margin_grad_mean': -0.5002588629722595, 'epsilon_dpo/beta_margin_grad_std': 0.006068441551178694, 'kl/beta': 0.09978172928094864, 'kl/avg_steps': 0.0625, 'epoch': 0.0}
|
|||
|
|
0%|▎ | 2/681 [00:05<29:49, 2.64s/it]
0%|▌ | 3/681 [00:08<30:30, 2.70s/it]
{'loss': 1.3817, 'grad_norm': 160.3519744873047, 'learning_rate': 1.4492753623188406e-08, 'rewards/chosen': 0.004386060871183872, 'rewards/rejected': -0.00041415661689825356, 'rewards/accuracies': 0.515625, 'rewards/margins': 0.004800218157470226, 'logps/chosen': -37.792213439941406, 'logps/rejected': -73.14955139160156, 'logps/ref_chosen': -37.83708190917969, 'logps/ref_rejected': -73.14408874511719, 'logits/chosen': -3.5040597915649414, 'logits/rejected': -3.4887866973876953, 'kl/p_epsilon_steps': 0.5, 'kl/n_epsilon_steps': 0.484375, 'epsilon_dpo/beta': 0.09971363842487335, 'epsilon_dpo/loss_margin_mean': 0.050338417291641235, 'epsilon_dpo/beta_margin_mean': 0.004800259135663509, 'epsilon_dpo/beta_margin_std': 0.028725432232022285, 'epsilon_dpo/beta_margin_grad_mean': -0.4988003969192505, 'epsilon_dpo/beta_margin_grad_std': 0.007179732900112867, 'kl/beta': 0.09971940517425537, 'kl/avg_steps': 0.015625, 'epoch': 0.0}
|
|||
|
|
0%|▌ | 3/681 [00:08<30:30, 2.70s/it]
1%|▋ | 4/681 [00:10<31:09, 2.76s/it]
{'loss': 1.3843, 'grad_norm': 155.65963745117188, 'learning_rate': 2.1739130434782606e-08, 'rewards/chosen': -0.000430062209488824, 'rewards/rejected': -0.0025970228016376495, 'rewards/accuracies': 0.53125, 'rewards/margins': 0.002166960621252656, 'logps/chosen': -43.339508056640625, 'logps/rejected': -93.78823852539062, 'logps/ref_chosen': -43.336036682128906, 'logps/ref_rejected': -93.7607650756836, 'logits/chosen': -3.468817710876465, 'logits/rejected': -3.508214235305786, 'kl/p_epsilon_steps': 0.546875, 'kl/n_epsilon_steps': 0.4375, 'epsilon_dpo/beta': 0.09960457682609558, 'epsilon_dpo/loss_margin_mean': 0.02400125563144684, 'epsilon_dpo/beta_margin_mean': 0.002166971331462264, 'epsilon_dpo/beta_margin_std': 0.029604651033878326, 'epsilon_dpo/beta_margin_grad_mean': -0.4994582235813141, 'epsilon_dpo/beta_margin_grad_std': 0.007399262860417366, 'kl/beta': 0.0997038260102272, 'kl/avg_steps': 0.109375, 'epoch': 0.01}
|
|||
|
|
1%|▋ | 4/681 [00:10<31:09, 2.76s/it]
1%|▊ | 5/681 [00:13<31:00, 2.75s/it]
{'loss': 1.3905, 'grad_norm': 179.6461639404297, 'learning_rate': 2.898550724637681e-08, 'rewards/chosen': -0.0026276579592376947, 'rewards/rejected': 0.0012790121836587787, 'rewards/accuracies': 0.484375, 'rewards/margins': -0.003906670026481152, 'logps/chosen': -32.93244934082031, 'logps/rejected': -90.29646301269531, 'logps/ref_chosen': -32.90675354003906, 'logps/ref_rejected': -90.30764770507812, 'logits/chosen': -3.436485767364502, 'logits/rejected': -3.4411563873291016, 'kl/p_epsilon_steps': 0.484375, 'kl/n_epsilon_steps': 0.515625, 'epsilon_dpo/beta': 0.09963598102331161, 'epsilon_dpo/loss_margin_mean': -0.036883413791656494, 'epsilon_dpo/beta_margin_mean': -0.003906682599335909, 'epsilon_dpo/beta_margin_std': 0.033521223813295364, 'epsilon_dpo/beta_margin_grad_mean': -0.5009759068489075, 'epsilon_dpo/beta_margin_grad_std': 0.008376221172511578, 'kl/beta': 0.09959489107131958, 'kl/avg_steps': -0.03125, 'epoch': 0.01}
|
|||
|
|
1%|▊ | 5/681 [00:13<31:00, 2.75s/it]
1%|█ | 6/681 [00:16<29:25, 2.62s/it]
{'loss': 1.3837, 'grad_norm': 168.01638793945312, 'learning_rate': 3.6231884057971014e-08, 'rewards/chosen': -0.0019228225573897362, 'rewards/rejected': -0.004670892842113972, 'rewards/accuracies': 0.53125, 'rewards/margins': 0.0027480702847242355, 'logps/chosen': -40.59587478637695, 'logps/rejected': -98.52117919921875, 'logps/ref_chosen': -40.57701110839844, 'logps/ref_rejected': -98.47296142578125, 'logits/chosen': -3.476027011871338, 'logits/rejected': -3.391662836074829, 'kl/p_epsilon_steps': 0.546875, 'kl/n_epsilon_steps': 0.453125, 'epsilon_dpo/beta': 0.09954258054494858, 'epsilon_dpo/loss_margin_mean': 0.029358193278312683, 'epsilon_dpo/beta_margin_mean': 0.002748197177425027, 'epsilon_dpo/beta_margin_std': 0.025408554822206497, 'epsilon_dpo/beta_margin_grad_mean': -0.4993135929107666, 'epsilon_dpo/beta_margin_grad_std': 0.006349037401378155, 'kl/beta': 0.09962602704763412, 'kl/avg_steps': 0.09375, 'epoch': 0.01}
|
|||
|
|
1%|█ | 6/681 [00:16<29:25, 2.62s/it]
1%|█▏ | 7/681 [00:18<28:56, 2.58s/it]
{'loss': 1.3862, 'grad_norm': 165.30645751953125, 'learning_rate': 4.347826086956521e-08, 'rewards/chosen': 0.002097110729664564, 'rewards/rejected': 0.0017947049345821142, 'rewards/accuracies': 0.515625, 'rewards/margins': 0.0003024058823939413, 'logps/chosen': -41.3924674987793, 'logps/rejected': -111.000244140625, 'logps/ref_chosen': -41.414642333984375, 'logps/ref_rejected': -111.01716613769531, 'logits/chosen': -3.4681951999664307, 'logits/rejected': -3.4873459339141846, 'kl/p_epsilon_steps': 0.515625, 'kl/n_epsilon_steps': 0.484375, 'epsilon_dpo/beta': 0.09951155632734299, 'epsilon_dpo/loss_margin_mean': 0.00525471568107605, 'epsilon_dpo/beta_margin_mean': 0.0003023834724444896, 'epsilon_dpo/beta_margin_std': 0.028604645282030106, 'epsilon_dpo/beta_margin_grad_mean': -0.49992430210113525, 'epsilon_dpo/beta_margin_grad_std': 0.007149491459131241, 'kl/beta': 0.09953271597623825, 'kl/avg_steps': 0.03125, 'epoch': 0.01}
|
|||
|
|
1%|█▏ | 7/681 [00:18<28:56, 2.58s/it]
1%|█▎ | 8/681 [00:21<28:51, 2.57s/it]
{'loss': 1.3941, 'grad_norm': 170.1224822998047, 'learning_rate': 5.0724637681159424e-08, 'rewards/chosen': -0.0024497562553733587, 'rewards/rejected': 0.005181857850402594, 'rewards/accuracies': 0.328125, 'rewards/margins': -0.007631613872945309, 'logps/chosen': -39.27941131591797, 'logps/rejected': -85.89848327636719, 'logps/ref_chosen': -39.25566482543945, 'logps/ref_rejected': -85.94947814941406, 'logits/chosen': -3.5701963901519775, 'logits/rejected': -3.46230411529541, 'kl/p_epsilon_steps': 0.328125, 'kl/n_epsilon_steps': 0.65625, 'epsilon_dpo/beta': 0.09983792901039124, 'epsilon_dpo/loss_margin_mean': -0.07474762201309204, 'epsilon_dpo/beta_margin_mean': -0.007631635759025812, 'epsilon_dpo/beta_margin_std': 0.02497241646051407, 'epsilon_dpo/beta_margin_grad_mean': -0.5019076466560364, 'epsilon_dpo/beta_margin_grad_std': 0.006241849157959223, 'kl/beta': 0.09950161725282669, 'kl/avg_steps': -0.328125, 'epoch': 0.01}
|
|||
|
|
1%|█▎ | 8/681 [00:21<28:51, 2.57s/it]
1%|█▌ | 9/681 [00:23<29:28, 2.63s/it]
{'loss': 1.3867, 'grad_norm': 179.10533142089844, 'learning_rate': 5.797101449275362e-08, 'rewards/chosen': -4.6280911192297935e-07, 'rewards/rejected': 0.00030112662352621555, 'rewards/accuracies': 0.453125, 'rewards/margins': -0.0003015893744304776, 'logps/chosen': -38.925682067871094, 'logps/rejected': -106.42868041992188, 'logps/ref_chosen': -38.9265251159668, 'logps/ref_rejected': -106.43075561523438, 'logits/chosen': -3.526092529296875, 'logits/rejected': -3.532437801361084, 'kl/p_epsilon_steps': 0.5, 'kl/n_epsilon_steps': 0.5, 'epsilon_dpo/beta': 0.09983916580677032, 'epsilon_dpo/loss_margin_mean': -0.0012376010417938232, 'epsilon_dpo/beta_margin_mean': -0.0003015802940353751, 'epsilon_dpo/beta_margin_std': 0.02405872382223606, 'epsilon_dpo/beta_margin_grad_mean': -0.5000754594802856, 'epsilon_dpo/beta_margin_grad_std': 0.006013626232743263, 'kl/beta': 0.09982918202877045, 'kl/avg_steps': 0.0, 'epoch': 0.01}
|
|||
|
|
1%|█▌ | 9/681 [00:23<29:28, 2.63s/it]
1%|█▋ | 10/681 [00:26<29:56, 2.68s/it]
{'loss': 1.3838, 'grad_norm': 161.84414672851562, 'learning_rate': 6.521739130434782e-08, 'rewards/chosen': 0.0009347056620754302, 'rewards/rejected': -0.0017286810325458646, 'rewards/accuracies': 0.578125, 'rewards/margins': 0.0026633867528289557, 'logps/chosen': -37.931678771972656, 'logps/rejected': -84.44198608398438, 'logps/ref_chosen': -37.94172668457031, 'logps/ref_rejected': -84.4234848022461, 'logits/chosen': -3.487415313720703, 'logits/rejected': -3.4311609268188477, 'kl/p_epsilon_steps': 0.546875, 'kl/n_epsilon_steps': 0.4375, 'epsilon_dpo/beta': 0.09972980618476868, 'epsilon_dpo/loss_margin_mean': 0.028553977608680725, 'epsilon_dpo/beta_margin_mean': 0.0026634030509740114, 'epsilon_dpo/beta_margin_std': 0.02231536991894245, 'epsilon_dpo/beta_margin_grad_mean': -0.49933433532714844, 'epsilon_dpo/beta_margin_grad_std': 0.005578126758337021, 'kl/beta': 0.09982918202877045, 'kl/avg_steps': 0.109375, 'epoch': 0.01}
|
|||
|
|
1%|█▋ | 10/681 [00:26<29:56, 2.68s/it]
2%|█▊ | 11/681 [00:29<30:11, 2.70s/it]
{'loss': 1.3775, 'grad_norm': 167.265380859375, 'learning_rate': 7.246376811594203e-08, 'rewards/chosen': 0.0010777543066069484, 'rewards/rejected': -0.007868202403187752, 'rewards/accuracies': 0.578125, 'rewards/margins': 0.008945956826210022, 'logps/chosen': -32.73089599609375, 'logps/rejected': -94.19029998779297, 'logps/ref_chosen': -32.742462158203125, 'logps/ref_rejected': -94.11013793945312, 'logits/chosen': -3.4978814125061035, 'logits/rejected': -3.5033230781555176, 'kl/p_epsilon_steps': 0.59375, 'kl/n_epsilon_steps': 0.40625, 'epsilon_dpo/beta': 0.09954308718442917, 'epsilon_dpo/loss_margin_mean': 0.09173570573329926, 'epsilon_dpo/beta_margin_mean': 0.008945857174694538, 'epsilon_dpo/beta_margin_std': 0.02535552717745304, 'epsilon_dpo/beta_margin_grad_mean': -0.49776411056518555, 'epsilon_dpo/beta_margin_grad_std': 0.006337402854114771, 'kl/beta': 0.09972011297941208, 'kl/avg_steps': 0.1875, 'epoch': 0.02}
|
|||
|
|
2%|█▊ | 11/681 [00:29<30:11, 2.70s/it]
2%|██ | 12/681 [00:32<30:01, 2.69s/it]
{'loss': 1.3817, 'grad_norm': 184.79959106445312, 'learning_rate': 7.971014492753623e-08, 'rewards/chosen': -0.0023162683937698603, 'rewards/rejected': -0.007148245815187693, 'rewards/accuracies': 0.625, 'rewards/margins': 0.004831977654248476, 'logps/chosen': -43.87688446044922, 'logps/rejected': -111.80323791503906, 'logps/ref_chosen': -43.85453796386719, 'logps/ref_rejected': -111.72984313964844, 'logits/chosen': -3.5019748210906982, 'logits/rejected': -3.507927417755127, 'kl/p_epsilon_steps': 0.625, 'kl/n_epsilon_steps': 0.375, 'epsilon_dpo/beta': 0.09929458051919937, 'epsilon_dpo/loss_margin_mean': 0.05105578899383545, 'epsilon_dpo/beta_margin_mean': 0.004831877537071705, 'epsilon_dpo/beta_margin_std': 0.03315826877951622, 'epsilon_dpo/beta_margin_grad_mean': -0.49879196286201477, 'epsilon_dpo/beta_margin_grad_std': 0.008286659605801105, 'kl/beta': 0.09953349083662033, 'kl/avg_steps': 0.25, 'epoch': 0.02}
|
|||
|
|
2%|██ | 12/681 [00:32<30:01, 2.69s/it]
2%|██▏ | 13/681 [00:34<30:03, 2.70s/it]
{'loss': 1.3832, 'grad_norm': 170.2859344482422, 'learning_rate': 8.695652173913042e-08, 'rewards/chosen': 0.0017041495302692056, 'rewards/rejected': -0.0015271796146407723, 'rewards/accuracies': 0.484375, 'rewards/margins': 0.0032313289120793343, 'logps/chosen': -41.653934478759766, 'logps/rejected': -91.1076889038086, 'logps/ref_chosen': -41.67176818847656, 'logps/ref_rejected': -91.09086608886719, 'logits/chosen': -3.540365219116211, 'logits/rejected': -3.4680285453796387, 'kl/p_epsilon_steps': 0.46875, 'kl/n_epsilon_steps': 0.53125, 'epsilon_dpo/beta': 0.09935726225376129, 'epsilon_dpo/loss_margin_mean': 0.034654468297958374, 'epsilon_dpo/beta_margin_mean': 0.00323132099583745, 'epsilon_dpo/beta_margin_std': 0.026057543233036995, 'epsilon_dpo/beta_margin_grad_mean': -0.49919238686561584, 'epsilon_dpo/beta_margin_grad_std': 0.006513380445539951, 'kl/beta': 0.09928527474403381, 'kl/avg_steps': -0.0625, 'epoch': 0.02}
|
|||
|
|
2%|██▏ | 13/681 [00:34<30:03, 2.70s/it]
2%|██▎ | 14/681 [00:37<29:32, 2.66s/it]
{'loss': 1.3668, 'grad_norm': 191.28602600097656, 'learning_rate': 9.420289855072464e-08, 'rewards/chosen': 0.0063568041659891605, 'rewards/rejected': -0.01342916302382946, 'rewards/accuracies': 0.734375, 'rewards/margins': 0.019785966724157333, 'logps/chosen': -32.60367202758789, 'logps/rejected': -111.98973846435547, 'logps/ref_chosen': -32.668601989746094, 'logps/ref_rejected': -111.8526611328125, 'logits/chosen': -3.5063905715942383, 'logits/rejected': -3.5407614707946777, 'kl/p_epsilon_steps': 0.71875, 'kl/n_epsilon_steps': 0.28125, 'epsilon_dpo/beta': 0.09892261028289795, 'epsilon_dpo/loss_margin_mean': 0.20201179385185242, 'epsilon_dpo/beta_margin_mean': 0.01978609710931778, 'epsilon_dpo/beta_margin_std': 0.03049391135573387, 'epsilon_dpo/beta_margin_grad_mean': -0.49505484104156494, 'epsilon_dpo/beta_margin_grad_std': 0.007620512507855892, 'kl/beta': 0.09934736788272858, 'kl/avg_steps': 0.4375, 'epoch': 0.02}
|
|||
|
|
2%|██▎ | 14/681 [00:37<29:32, 2.66s/it]
2%|██▌ | 15/681 [00:40<29:35, 2.67s/it]
{'loss': 1.3699, 'grad_norm': 144.098388671875, 'learning_rate': 1.0144927536231885e-07, 'rewards/chosen': -0.0012117127189412713, 'rewards/rejected': -0.01782957650721073, 'rewards/accuracies': 0.734375, 'rewards/margins': 0.01661786250770092, 'logps/chosen': -41.79482650756836, 'logps/rejected': -86.60433197021484, 'logps/ref_chosen': -41.78295135498047, 'logps/ref_rejected': -86.42213439941406, 'logits/chosen': -3.4299309253692627, 'logits/rejected': -3.4826550483703613, 'kl/p_epsilon_steps': 0.734375, 'kl/n_epsilon_steps': 0.25, 'epsilon_dpo/beta': 0.09844519197940826, 'epsilon_dpo/loss_margin_mean': 0.17031680047512054, 'epsilon_dpo/beta_margin_mean': 0.016617843881249428, 'epsilon_dpo/beta_margin_std': 0.025397835299372673, 'epsilon_dpo/beta_margin_grad_mean': -0.4958465099334717, 'epsilon_dpo/beta_margin_grad_std': 0.006347531918436289, 'kl/beta': 0.09891461580991745, 'kl/avg_steps': 0.484375, 'epoch': 0.02}
|
|||
|
|
2%|██▌ | 15/681 [00:40<29:35, 2.67s/it]
2%|██▋ | 16/681 [00:42<29:29, 2.66s/it]
{'loss': 1.37, 'grad_norm': 150.39041137695312, 'learning_rate': 1.0869565217391303e-07, 'rewards/chosen': 0.0014686340000480413, 'rewards/rejected': -0.015126017853617668, 'rewards/accuracies': 0.75, 'rewards/margins': 0.016594652086496353, 'logps/chosen': -40.885353088378906, 'logps/rejected': -89.05500793457031, 'logps/ref_chosen': -40.9011116027832, 'logps/ref_rejected': -88.89961242675781, 'logits/chosen': -3.477813959121704, 'logits/rejected': -3.4168307781219482, 'kl/p_epsilon_steps': 0.75, 'kl/n_epsilon_steps': 0.25, 'epsilon_dpo/beta': 0.09795541316270828, 'epsilon_dpo/loss_margin_mean': 0.1711507886648178, 'epsilon_dpo/beta_margin_mean': 0.016594676300883293, 'epsilon_dpo/beta_margin_std': 0.029557235538959503, 'epsilon_dpo/beta_margin_grad_mean': -0.4958525598049164, 'epsilon_dpo/beta_margin_grad_std': 0.007386185694485903, 'kl/beta': 0.09843780845403671, 'kl/avg_steps': 0.5, 'epoch': 0.02}
|
|||
|
|
2%|██▋ | 16/681 [00:42<29:29, 2.66s/it]
2%|██▊ | 17/681 [00:45<29:14, 2.64s/it]
{'loss': 1.3536, 'grad_norm': 171.44161987304688, 'learning_rate': 1.1594202898550725e-07, 'rewards/chosen': 0.00957987830042839, 'rewards/rejected': -0.023779571056365967, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.03335944563150406, 'logps/chosen': -38.45612335205078, 'logps/rejected': -91.90687561035156, 'logps/ref_chosen': -38.555274963378906, 'logps/ref_rejected': -91.66143798828125, 'logits/chosen': -3.526141405105591, 'logits/rejected': -3.4046664237976074, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.09734562784433365, 'epsilon_dpo/loss_margin_mean': 0.34458500146865845, 'epsilon_dpo/beta_margin_mean': 0.033359427005052567, 'epsilon_dpo/beta_margin_std': 0.037631068378686905, 'epsilon_dpo/beta_margin_grad_mean': -0.49166470766067505, 'epsilon_dpo/beta_margin_grad_std': 0.009398871101439, 'kl/beta': 0.09794806689023972, 'kl/avg_steps': 0.625, 'epoch': 0.02}
|
|||
|
|
2%|██▊ | 17/681 [00:45<29:14, 2.64s/it]
3%|███ | 18/681 [00:47<29:11, 2.64s/it]
{'loss': 1.3496, 'grad_norm': 174.4280242919922, 'learning_rate': 1.2318840579710146e-07, 'rewards/chosen': 0.008510860614478588, 'rewards/rejected': -0.028904292732477188, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.03741515427827835, 'logps/chosen': -26.462953567504883, 'logps/rejected': -82.455078125, 'logps/ref_chosen': -26.55130386352539, 'logps/ref_rejected': -82.15496063232422, 'logits/chosen': -3.457716464996338, 'logits/rejected': -3.3878774642944336, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.09655846655368805, 'epsilon_dpo/loss_margin_mean': 0.3884708285331726, 'epsilon_dpo/beta_margin_mean': 0.03741518035531044, 'epsilon_dpo/beta_margin_std': 0.03761782497167587, 'epsilon_dpo/beta_margin_grad_mean': -0.4906516373157501, 'epsilon_dpo/beta_margin_grad_std': 0.009394010528922081, 'kl/beta': 0.0973396971821785, 'kl/avg_steps': 0.8125, 'epoch': 0.03}
|
|||
|
|
3%|███ | 18/681 [00:47<29:11, 2.64s/it]
3%|███▏ | 19/681 [00:50<28:50, 2.61s/it]
{'loss': 1.3362, 'grad_norm': 158.5471649169922, 'learning_rate': 1.3043478260869563e-07, 'rewards/chosen': 0.009450599551200867, 'rewards/rejected': -0.04208826646208763, 'rewards/accuracies': 0.828125, 'rewards/margins': 0.0515388660132885, 'logps/chosen': -51.44847869873047, 'logps/rejected': -96.42372131347656, 'logps/ref_chosen': -51.548377990722656, 'logps/ref_rejected': -95.98385620117188, 'logits/chosen': -3.5245118141174316, 'logits/rejected': -3.4201385974884033, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.0959613099694252, 'epsilon_dpo/loss_margin_mean': 0.5397706031799316, 'epsilon_dpo/beta_margin_mean': 0.05153882876038551, 'epsilon_dpo/beta_margin_std': 0.05628020688891411, 'epsilon_dpo/beta_margin_grad_mean': -0.4871312379837036, 'epsilon_dpo/beta_margin_grad_std': 0.014039521105587482, 'kl/beta': 0.0965551808476448, 'kl/avg_steps': 0.625, 'epoch': 0.03}
|
|||
|
|
3%|███▏ | 19/681 [00:50<28:50, 2.61s/it]
3%|███▍ | 20/681 [00:53<28:56, 2.63s/it]
{'loss': 1.3324, 'grad_norm': 148.6184844970703, 'learning_rate': 1.3768115942028986e-07, 'rewards/chosen': 0.012646486982703209, 'rewards/rejected': -0.042639512568712234, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.055285997688770294, 'logps/chosen': -32.44221878051758, 'logps/rejected': -84.17327880859375, 'logps/ref_chosen': -32.57563781738281, 'logps/ref_rejected': -83.72441101074219, 'logits/chosen': -3.5266153812408447, 'logits/rejected': -3.454347848892212, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.09518533945083618, 'epsilon_dpo/loss_margin_mean': 0.5822827816009521, 'epsilon_dpo/beta_margin_mean': 0.05528602749109268, 'epsilon_dpo/beta_margin_std': 0.05034392327070236, 'epsilon_dpo/beta_margin_grad_mean': -0.4861924350261688, 'epsilon_dpo/beta_margin_grad_std': 0.01256165187805891, 'kl/beta': 0.09595546126365662, 'kl/avg_steps': 0.8125, 'epoch': 0.03}
|
|||
|
|
3%|███▍ | 20/681 [00:53<28:56, 2.63s/it]
3%|███▌ | 21/681 [00:55<28:40, 2.61s/it]
{'loss': 1.3294, 'grad_norm': 138.44097900390625, 'learning_rate': 1.4492753623188405e-07, 'rewards/chosen': 0.007934953086078167, 'rewards/rejected': -0.05056622251868248, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.05850117653608322, 'logps/chosen': -36.161651611328125, 'logps/rejected': -83.22535705566406, 'logps/ref_chosen': -36.24628448486328, 'logps/ref_rejected': -82.68882751464844, 'logits/chosen': -3.416736602783203, 'logits/rejected': -3.4210104942321777, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.09435869753360748, 'epsilon_dpo/loss_margin_mean': 0.6211739182472229, 'epsilon_dpo/beta_margin_mean': 0.058501191437244415, 'epsilon_dpo/beta_margin_std': 0.054356660693883896, 'epsilon_dpo/beta_margin_grad_mean': -0.485392689704895, 'epsilon_dpo/beta_margin_grad_std': 0.01355255488306284, 'kl/beta': 0.09518210589885712, 'kl/avg_steps': 0.875, 'epoch': 0.03}
|
|||
|
|
3%|███▌ | 21/681 [00:55<28:40, 2.61s/it]
3%|███▋ | 22/681 [00:58<29:09, 2.66s/it]
{'loss': 1.3155, 'grad_norm': 163.26019287109375, 'learning_rate': 1.5217391304347825e-07, 'rewards/chosen': 0.007986144162714481, 'rewards/rejected': -0.06498903036117554, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.07297517359256744, 'logps/chosen': -44.6229248046875, 'logps/rejected': -104.12324523925781, 'logps/ref_chosen': -44.70884704589844, 'logps/ref_rejected': -103.42787170410156, 'logits/chosen': -3.4850616455078125, 'logits/rejected': -3.4672698974609375, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.09351073950529099, 'epsilon_dpo/loss_margin_mean': 0.7812974452972412, 'epsilon_dpo/beta_margin_mean': 0.07297518849372864, 'epsilon_dpo/beta_margin_std': 0.05880572646856308, 'epsilon_dpo/beta_margin_grad_mean': -0.48178407549858093, 'epsilon_dpo/beta_margin_grad_std': 0.01465072762221098, 'kl/beta': 0.0943564921617508, 'kl/avg_steps': 0.90625, 'epoch': 0.03}
|
|||
|
|
3%|███▋ | 22/681 [00:58<29:09, 2.66s/it]
3%|███▉ | 23/681 [01:01<29:18, 2.67s/it]
{'loss': 1.3054, 'grad_norm': 153.97116088867188, 'learning_rate': 1.5942028985507245e-07, 'rewards/chosen': 0.013693436048924923, 'rewards/rejected': -0.07020558416843414, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.08389902114868164, 'logps/chosen': -40.120094299316406, 'logps/rejected': -86.60855102539062, 'logps/ref_chosen': -40.26862335205078, 'logps/ref_rejected': -85.85059356689453, 'logits/chosen': -3.4643330574035645, 'logits/rejected': -3.442781686782837, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.09270013123750687, 'epsilon_dpo/loss_margin_mean': 0.9064887762069702, 'epsilon_dpo/beta_margin_mean': 0.08389898389577866, 'epsilon_dpo/beta_margin_std': 0.07160933315753937, 'epsilon_dpo/beta_margin_grad_mean': -0.47907260060310364, 'epsilon_dpo/beta_margin_grad_std': 0.01780831441283226, 'kl/beta': 0.09350906312465668, 'kl/avg_steps': 0.875, 'epoch': 0.03}
|
|||
|
|
3%|███▉ | 23/681 [01:01<29:18, 2.67s/it]
4%|████ | 24/681 [01:03<29:05, 2.66s/it]
{'loss': 1.2883, 'grad_norm': 161.09317016601562, 'learning_rate': 1.6666666666666665e-07, 'rewards/chosen': 0.015648098662495613, 'rewards/rejected': -0.08636436611413956, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.10201247036457062, 'logps/chosen': -27.038997650146484, 'logps/rejected': -107.220458984375, 'logps/ref_chosen': -27.20970916748047, 'logps/ref_rejected': -106.27947998046875, 'logits/chosen': -3.4035227298736572, 'logits/rejected': -3.4438557624816895, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0919250100851059, 'epsilon_dpo/loss_margin_mean': 1.1116865873336792, 'epsilon_dpo/beta_margin_mean': 0.10201232135295868, 'epsilon_dpo/beta_margin_std': 0.07649125903844833, 'epsilon_dpo/beta_margin_grad_mean': -0.4745597541332245, 'epsilon_dpo/beta_margin_grad_std': 0.019036216661334038, 'kl/beta': 0.09269795566797256, 'kl/avg_steps': 0.84375, 'epoch': 0.04}
|
|||
|
|
4%|████ | 24/681 [01:03<29:05, 2.66s/it]
4%|████▏ | 25/681 [01:06<29:23, 2.69s/it]
{'loss': 1.2937, 'grad_norm': 128.98117065429688, 'learning_rate': 1.7391304347826085e-07, 'rewards/chosen': 0.004450969398021698, 'rewards/rejected': -0.0927439033985138, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.09719488024711609, 'logps/chosen': -36.42045593261719, 'logps/rejected': -94.91465759277344, 'logps/ref_chosen': -36.47064208984375, 'logps/ref_rejected': -93.89593505859375, 'logits/chosen': -3.487727403640747, 'logits/rejected': -3.466651439666748, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.09127078950405121, 'epsilon_dpo/loss_margin_mean': 1.0689027309417725, 'epsilon_dpo/beta_margin_mean': 0.09719490259885788, 'epsilon_dpo/beta_margin_std': 0.09393724054098129, 'epsilon_dpo/beta_margin_grad_mean': -0.47578126192092896, 'epsilon_dpo/beta_margin_grad_std': 0.023359432816505432, 'kl/beta': 0.09192235767841339, 'kl/avg_steps': 0.71875, 'epoch': 0.04}
|
|||
|
|
4%|████▏ | 25/681 [01:06<29:23, 2.69s/it]
4%|████▍ | 26/681 [01:08<28:26, 2.61s/it]
{'loss': 1.2385, 'grad_norm': 150.258056640625, 'learning_rate': 1.8115942028985507e-07, 'rewards/chosen': 0.010786900296807289, 'rewards/rejected': -0.15054798126220703, 'rewards/accuracies': 0.875, 'rewards/margins': 0.16133487224578857, 'logps/chosen': -39.704803466796875, 'logps/rejected': -110.67877197265625, 'logps/ref_chosen': -39.82624816894531, 'logps/ref_rejected': -109.0130615234375, 'logits/chosen': -3.4571170806884766, 'logits/rejected': -3.5464980602264404, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0905909463763237, 'epsilon_dpo/loss_margin_mean': 1.7871633768081665, 'epsilon_dpo/beta_margin_mean': 0.16133488714694977, 'epsilon_dpo/beta_margin_std': 0.1687132865190506, 'epsilon_dpo/beta_margin_grad_mean': -0.4600984454154968, 'epsilon_dpo/beta_margin_grad_std': 0.04119768738746643, 'kl/beta': 0.09126638621091843, 'kl/avg_steps': 0.75, 'epoch': 0.04}
|
|||
|
|
4%|████▍ | 26/681 [01:08<28:26, 2.61s/it]
4%|████▌ | 27/681 [01:11<28:08, 2.58s/it]
{'loss': 1.2104, 'grad_norm': 152.13011169433594, 'learning_rate': 1.8840579710144927e-07, 'rewards/chosen': 0.012909766286611557, 'rewards/rejected': -0.17539237439632416, 'rewards/accuracies': 0.984375, 'rewards/margins': 0.18830212950706482, 'logps/chosen': -19.790260314941406, 'logps/rejected': -109.28077697753906, 'logps/ref_chosen': -19.93426513671875, 'logps/ref_rejected': -107.32525634765625, 'logits/chosen': -3.417879104614258, 'logits/rejected': -3.4293429851531982, 'kl/p_epsilon_steps': 0.984375, 'kl/n_epsilon_steps': 0.015625, 'epsilon_dpo/beta': 0.08971839398145676, 'epsilon_dpo/loss_margin_mean': 2.099519729614258, 'epsilon_dpo/beta_margin_mean': 0.1883021891117096, 'epsilon_dpo/beta_margin_std': 0.1207134947180748, 'epsilon_dpo/beta_margin_grad_mean': -0.4532409906387329, 'epsilon_dpo/beta_margin_grad_std': 0.02980455383658409, 'kl/beta': 0.09058698266744614, 'kl/avg_steps': 0.96875, 'epoch': 0.04}
|
|||
|
|
4%|████▌ | 27/681 [01:11<28:08, 2.58s/it]
4%|████▋ | 28/681 [01:14<28:14, 2.60s/it]
{'loss': 1.2081, 'grad_norm': 144.26730346679688, 'learning_rate': 1.9565217391304347e-07, 'rewards/chosen': 0.02439035288989544, 'rewards/rejected': -0.1676751971244812, 'rewards/accuracies': 0.96875, 'rewards/margins': 0.19206556677818298, 'logps/chosen': -43.32648849487305, 'logps/rejected': -98.03630065917969, 'logps/ref_chosen': -43.6025390625, 'logps/ref_rejected': -96.1494140625, 'logits/chosen': -3.5261058807373047, 'logits/rejected': -3.475046157836914, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.08888562768697739, 'epsilon_dpo/loss_margin_mean': 2.162940263748169, 'epsilon_dpo/beta_margin_mean': 0.1920655220746994, 'epsilon_dpo/beta_margin_std': 0.13740301132202148, 'epsilon_dpo/beta_margin_grad_mean': -0.4523911476135254, 'epsilon_dpo/beta_margin_grad_std': 0.03366325423121452, 'kl/beta': 0.08971784263849258, 'kl/avg_steps': 0.9375, 'epoch': 0.04}
|
|||
|
|
4%|████▋ | 28/681 [01:14<28:14, 2.60s/it]
4%|████▉ | 29/681 [01:16<27:31, 2.53s/it]
{'loss': 1.1716, 'grad_norm': 141.24661254882812, 'learning_rate': 2.028985507246377e-07, 'rewards/chosen': 0.03065740317106247, 'rewards/rejected': -0.20474430918693542, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.2354017198085785, 'logps/chosen': -32.09503936767578, 'logps/rejected': -104.30924224853516, 'logps/ref_chosen': -32.44408416748047, 'logps/ref_rejected': -101.98307037353516, 'logits/chosen': -3.4727797508239746, 'logits/rejected': -3.4470648765563965, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.08808783441781998, 'epsilon_dpo/loss_margin_mean': 2.675217866897583, 'epsilon_dpo/beta_margin_mean': 0.23540179431438446, 'epsilon_dpo/beta_margin_std': 0.1678141951560974, 'epsilon_dpo/beta_margin_grad_mean': -0.4418832063674927, 'epsilon_dpo/beta_margin_grad_std': 0.04080166295170784, 'kl/beta': 0.08888454735279083, 'kl/avg_steps': 0.90625, 'epoch': 0.04}
|
|||
|
|
4%|████▉ | 29/681 [01:16<27:31, 2.53s/it]
4%|█████ | 30/681 [01:19<27:47, 2.56s/it]
{'loss': 1.1744, 'grad_norm': 139.83612060546875, 'learning_rate': 2.1014492753623187e-07, 'rewards/chosen': 0.0018627983517944813, 'rewards/rejected': -0.23203766345977783, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.23390044271945953, 'logps/chosen': -39.259483337402344, 'logps/rejected': -106.5517807006836, 'logps/ref_chosen': -39.2830810546875, 'logps/ref_rejected': -103.8922119140625, 'logits/chosen': -3.5117549896240234, 'logits/rejected': -3.389063835144043, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.08729670941829681, 'epsilon_dpo/loss_margin_mean': 2.6831705570220947, 'epsilon_dpo/beta_margin_mean': 0.23390042781829834, 'epsilon_dpo/beta_margin_std': 0.1849268078804016, 'epsilon_dpo/beta_margin_grad_mean': -0.44231948256492615, 'epsilon_dpo/beta_margin_grad_std': 0.04499894008040428, 'kl/beta': 0.08808626234531403, 'kl/avg_steps': 0.90625, 'epoch': 0.04}
|
|||
|
|
4%|█████ | 30/681 [01:19<27:47, 2.56s/it]
5%|█████▏ | 31/681 [01:21<27:57, 2.58s/it]
{'loss': 1.1824, 'grad_norm': 121.26469421386719, 'learning_rate': 2.1739130434782607e-07, 'rewards/chosen': 0.02715596929192543, 'rewards/rejected': -0.19907742738723755, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.22623339295387268, 'logps/chosen': -35.737491607666016, 'logps/rejected': -80.65327453613281, 'logps/ref_chosen': -36.05577850341797, 'logps/ref_rejected': -78.35195922851562, 'logits/chosen': -3.4779539108276367, 'logits/rejected': -3.431878089904785, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.08656726032495499, 'epsilon_dpo/loss_margin_mean': 2.6196024417877197, 'epsilon_dpo/beta_margin_mean': 0.22623342275619507, 'epsilon_dpo/beta_margin_std': 0.1985906958580017, 'epsilon_dpo/beta_margin_grad_mean': -0.44433295726776123, 'epsilon_dpo/beta_margin_grad_std': 0.04770767688751221, 'kl/beta': 0.08729515224695206, 'kl/avg_steps': 0.84375, 'epoch': 0.05}
|
|||
|
|
5%|█████▏ | 31/681 [01:21<27:57, 2.58s/it]
5%|█████▍ | 32/681 [01:24<27:43, 2.56s/it]
{'loss': 1.1284, 'grad_norm': 131.14877319335938, 'learning_rate': 2.2463768115942027e-07, 'rewards/chosen': 0.023832818493247032, 'rewards/rejected': -0.2682601809501648, 'rewards/accuracies': 0.96875, 'rewards/margins': 0.2920929789543152, 'logps/chosen': -32.82677459716797, 'logps/rejected': -98.60302734375, 'logps/ref_chosen': -33.10527420043945, 'logps/ref_rejected': -95.47318267822266, 'logits/chosen': -3.4760398864746094, 'logits/rejected': -3.5285589694976807, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.08576179295778275, 'epsilon_dpo/loss_margin_mean': 3.4083411693573, 'epsilon_dpo/beta_margin_mean': 0.292092889547348, 'epsilon_dpo/beta_margin_std': 0.2318342626094818, 'epsilon_dpo/beta_margin_grad_mean': -0.42864087224006653, 'epsilon_dpo/beta_margin_grad_std': 0.05508127063512802, 'kl/beta': 0.08656476438045502, 'kl/avg_steps': 0.9375, 'epoch': 0.05}
|
|||
|
|
5%|█████▍ | 32/681 [01:24<27:43, 2.56s/it]
5%|█████▌ | 33/681 [01:26<27:33, 2.55s/it]
{'loss': 1.1399, 'grad_norm': 127.04460906982422, 'learning_rate': 2.318840579710145e-07, 'rewards/chosen': 0.00047481246292591095, 'rewards/rejected': -0.2803095579147339, 'rewards/accuracies': 0.96875, 'rewards/margins': 0.28078436851501465, 'logps/chosen': -40.38764953613281, 'logps/rejected': -97.78340148925781, 'logps/ref_chosen': -40.39752960205078, 'logps/ref_rejected': -94.48348999023438, 'logits/chosen': -3.5147736072540283, 'logits/rejected': -3.4672505855560303, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.08496525138616562, 'epsilon_dpo/loss_margin_mean': 3.309786796569824, 'epsilon_dpo/beta_margin_mean': 0.2807844281196594, 'epsilon_dpo/beta_margin_std': 0.24786221981048584, 'epsilon_dpo/beta_margin_grad_mean': -0.43119847774505615, 'epsilon_dpo/beta_margin_grad_std': 0.058498919010162354, 'kl/beta': 0.08576075732707977, 'kl/avg_steps': 0.9375, 'epoch': 0.05}
|
|||
|
|
5%|█████▌ | 33/681 [01:26<27:33, 2.55s/it]
5%|█████▋ | 34/681 [01:29<27:18, 2.53s/it]
{'loss': 1.1152, 'grad_norm': 127.52790069580078, 'learning_rate': 2.391304347826087e-07, 'rewards/chosen': -0.0053707570768892765, 'rewards/rejected': -0.31881266832351685, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.3134418725967407, 'logps/chosen': -35.16386413574219, 'logps/rejected': -110.49327850341797, 'logps/ref_chosen': -35.10262680053711, 'logps/ref_rejected': -106.70514678955078, 'logits/chosen': -3.4469704627990723, 'logits/rejected': -3.5294151306152344, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.08430886268615723, 'epsilon_dpo/loss_margin_mean': 3.726898193359375, 'epsilon_dpo/beta_margin_mean': 0.3134419322013855, 'epsilon_dpo/beta_margin_std': 0.27257490158081055, 'epsilon_dpo/beta_margin_grad_mean': -0.42384782433509827, 'epsilon_dpo/beta_margin_grad_std': 0.06483103334903717, 'kl/beta': 0.08496421575546265, 'kl/avg_steps': 0.78125, 'epoch': 0.05}
|
|||
|
|
5%|█████▋ | 34/681 [01:29<27:18, 2.53s/it]
5%|█████▉ | 35/681 [01:31<27:08, 2.52s/it]
{'loss': 1.0629, 'grad_norm': 120.23053741455078, 'learning_rate': 2.463768115942029e-07, 'rewards/chosen': -0.019653314724564552, 'rewards/rejected': -0.4028552770614624, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.3832019567489624, 'logps/chosen': -34.6436767578125, 'logps/rejected': -116.70634460449219, 'logps/ref_chosen': -34.41180419921875, 'logps/ref_rejected': -111.88399505615234, 'logits/chosen': -3.4881181716918945, 'logits/rejected': -3.5153722763061523, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0835762619972229, 'epsilon_dpo/loss_margin_mean': 4.590485095977783, 'epsilon_dpo/beta_margin_mean': 0.3832019567489624, 'epsilon_dpo/beta_margin_std': 0.3155387043952942, 'epsilon_dpo/beta_margin_grad_mean': -0.4079773724079132, 'epsilon_dpo/beta_margin_grad_std': 0.07277688384056091, 'kl/beta': 0.08430557698011398, 'kl/avg_steps': 0.875, 'epoch': 0.05}
|
|||
|
|
5%|█████▉ | 35/681 [01:31<27:08, 2.52s/it]
5%|██████ | 36/681 [01:34<26:50, 2.50s/it]
{'loss': 1.0334, 'grad_norm': 105.79562377929688, 'learning_rate': 2.536231884057971e-07, 'rewards/chosen': 0.012363580986857414, 'rewards/rejected': -0.428993821144104, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.4413573741912842, 'logps/chosen': -32.58708572387695, 'logps/rejected': -95.34486389160156, 'logps/ref_chosen': -32.743473052978516, 'logps/ref_rejected': -90.1633529663086, 'logits/chosen': -3.4250617027282715, 'logits/rejected': -3.506716251373291, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.08292967081069946, 'epsilon_dpo/loss_margin_mean': 5.337898254394531, 'epsilon_dpo/beta_margin_mean': 0.4413573443889618, 'epsilon_dpo/beta_margin_std': 0.41608670353889465, 'epsilon_dpo/beta_margin_grad_mean': -0.3959466516971588, 'epsilon_dpo/beta_margin_grad_std': 0.09478563815355301, 'kl/beta': 0.08357430249452591, 'kl/avg_steps': 0.78125, 'epoch': 0.05}
|
|||
|
|
5%|██████ | 36/681 [01:34<26:50, 2.50s/it]
5%|██████▏ | 37/681 [01:36<26:15, 2.45s/it]
{'loss': 1.0351, 'grad_norm': 100.5900650024414, 'learning_rate': 2.6086956521739126e-07, 'rewards/chosen': -0.025536730885505676, 'rewards/rejected': -0.46323513984680176, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.4376984238624573, 'logps/chosen': -40.18016052246094, 'logps/rejected': -89.59291076660156, 'logps/ref_chosen': -39.88025665283203, 'logps/ref_rejected': -83.95890808105469, 'logits/chosen': -3.4775028228759766, 'logits/rejected': -3.44918155670166, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.08228681236505508, 'epsilon_dpo/loss_margin_mean': 5.334095478057861, 'epsilon_dpo/beta_margin_mean': 0.4376983940601349, 'epsilon_dpo/beta_margin_std': 0.4152112901210785, 'epsilon_dpo/beta_margin_grad_mean': -0.3971908390522003, 'epsilon_dpo/beta_margin_grad_std': 0.09092327207326889, 'kl/beta': 0.082926444709301, 'kl/avg_steps': 0.78125, 'epoch': 0.05}
|
|||
|
|
5%|██████▏ | 37/681 [01:36<26:15, 2.45s/it]
6%|██████▍ | 38/681 [01:38<26:00, 2.43s/it]
{'loss': 0.9294, 'grad_norm': 99.69669342041016, 'learning_rate': 2.681159420289855e-07, 'rewards/chosen': -0.0062013790011405945, 'rewards/rejected': -0.6268683671951294, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.6206669807434082, 'logps/chosen': -33.92012405395508, 'logps/rejected': -112.65040588378906, 'logps/ref_chosen': -33.85154342651367, 'logps/ref_rejected': -104.96053314208984, 'logits/chosen': -3.436084270477295, 'logits/rejected': -3.5204224586486816, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.08167463541030884, 'epsilon_dpo/loss_margin_mean': 7.621293067932129, 'epsilon_dpo/beta_margin_mean': 0.6206669211387634, 'epsilon_dpo/beta_margin_std': 0.5732053518295288, 'epsilon_dpo/beta_margin_grad_mean': -0.36130231618881226, 'epsilon_dpo/beta_margin_grad_std': 0.11581370234489441, 'kl/beta': 0.08228360116481781, 'kl/avg_steps': 0.75, 'epoch': 0.06}
|
|||
|
|
6%|██████▍ | 38/681 [01:38<26:00, 2.43s/it]
6%|██████▌ | 39/681 [01:41<26:02, 2.43s/it]
{'loss': 0.8922, 'grad_norm': 102.14472198486328, 'learning_rate': 2.753623188405797e-07, 'rewards/chosen': 0.035918496549129486, 'rewards/rejected': -0.6091995239257812, 'rewards/accuracies': 0.96875, 'rewards/margins': 0.645117998123169, 'logps/chosen': -31.436540603637695, 'logps/rejected': -91.78189086914062, 'logps/ref_chosen': -31.883747100830078, 'logps/ref_rejected': -84.24908447265625, 'logits/chosen': -3.435272216796875, 'logits/rejected': -3.441884994506836, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.0809134915471077, 'epsilon_dpo/loss_margin_mean': 7.98002290725708, 'epsilon_dpo/beta_margin_mean': 0.6451180577278137, 'epsilon_dpo/beta_margin_std': 0.48367542028427124, 'epsilon_dpo/beta_margin_grad_mean': -0.35245731472969055, 'epsilon_dpo/beta_margin_grad_std': 0.09772588312625885, 'kl/beta': 0.08167106658220291, 'kl/avg_steps': 0.9375, 'epoch': 0.06}
|
|||
|
|
6%|██████▌ | 39/681 [01:41<26:02, 2.43s/it]
6%|██████▊ | 40/681 [01:43<25:58, 2.43s/it]
{'loss': 0.9118, 'grad_norm': 94.2869644165039, 'learning_rate': 2.8260869565217386e-07, 'rewards/chosen': -0.018150903284549713, 'rewards/rejected': -0.654141366481781, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.6359904408454895, 'logps/chosen': -36.815399169921875, 'logps/rejected': -89.08628845214844, 'logps/ref_chosen': -36.59412384033203, 'logps/ref_rejected': -80.92609405517578, 'logits/chosen': -3.48494291305542, 'logits/rejected': -3.507589101791382, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.08031369745731354, 'epsilon_dpo/loss_margin_mean': 7.938919544219971, 'epsilon_dpo/beta_margin_mean': 0.6359905004501343, 'epsilon_dpo/beta_margin_std': 0.5488929152488708, 'epsilon_dpo/beta_margin_grad_mean': -0.35732436180114746, 'epsilon_dpo/beta_margin_grad_std': 0.10807473212480545, 'kl/beta': 0.0809125155210495, 'kl/avg_steps': 0.75, 'epoch': 0.06}
|
|||
|
|
6%|██████▊ | 40/681 [01:43<25:58, 2.43s/it]
6%|██████▉ | 41/681 [01:46<25:37, 2.40s/it]
{'loss': 0.889, 'grad_norm': 101.93110656738281, 'learning_rate': 2.898550724637681e-07, 'rewards/chosen': -0.10497994720935822, 'rewards/rejected': -0.8672504425048828, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.762270450592041, 'logps/chosen': -41.283260345458984, 'logps/rejected': -116.42434692382812, 'logps/ref_chosen': -39.986053466796875, 'logps/ref_rejected': -105.53334045410156, 'logits/chosen': -3.483686923980713, 'logits/rejected': -3.4968438148498535, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.07975336164236069, 'epsilon_dpo/loss_margin_mean': 9.593799591064453, 'epsilon_dpo/beta_margin_mean': 0.762270450592041, 'epsilon_dpo/beta_margin_std': 0.7963690161705017, 'epsilon_dpo/beta_margin_grad_mean': -0.3376636803150177, 'epsilon_dpo/beta_margin_grad_std': 0.1441744863986969, 'kl/beta': 0.08031018823385239, 'kl/avg_steps': 0.703125, 'epoch': 0.06}
|
|||
|
|
6%|██████▉ | 41/681 [01:46<25:37, 2.40s/it]
6%|███████ | 42/681 [01:48<26:24, 2.48s/it]
{'loss': 0.8297, 'grad_norm': 98.95552062988281, 'learning_rate': 2.971014492753623e-07, 'rewards/chosen': -0.1441619098186493, 'rewards/rejected': -1.0115324258804321, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.86737060546875, 'logps/chosen': -47.57762145996094, 'logps/rejected': -130.83279418945312, 'logps/ref_chosen': -45.769351959228516, 'logps/ref_rejected': -118.03570556640625, 'logits/chosen': -3.436079502105713, 'logits/rejected': -3.551501989364624, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.07915924489498138, 'epsilon_dpo/loss_margin_mean': 10.988822937011719, 'epsilon_dpo/beta_margin_mean': 0.8673705458641052, 'epsilon_dpo/beta_margin_std': 0.8684898018836975, 'epsilon_dpo/beta_margin_grad_mean': -0.3248225450515747, 'epsilon_dpo/beta_margin_grad_std': 0.14031846821308136, 'kl/beta': 0.0797494500875473, 'kl/avg_steps': 0.75, 'epoch': 0.06}
|
|||
|
|
6%|███████ | 42/681 [01:48<26:24, 2.48s/it]
6%|███████▎ | 43/681 [01:51<26:49, 2.52s/it]
{'loss': 0.7552, 'grad_norm': 86.0045166015625, 'learning_rate': 3.043478260869565e-07, 'rewards/chosen': -0.010491969995200634, 'rewards/rejected': -0.9838355779647827, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.9733436703681946, 'logps/chosen': -36.81052017211914, 'logps/rejected': -117.46549224853516, 'logps/ref_chosen': -36.684478759765625, 'logps/ref_rejected': -104.91730499267578, 'logits/chosen': -3.4531750679016113, 'logits/rejected': -3.502927303314209, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.07852049171924591, 'epsilon_dpo/loss_margin_mean': 12.422151565551758, 'epsilon_dpo/beta_margin_mean': 0.9733436703681946, 'epsilon_dpo/beta_margin_std': 0.7786263227462769, 'epsilon_dpo/beta_margin_grad_mean': -0.29879364371299744, 'epsilon_dpo/beta_margin_grad_std': 0.14372758567333221, 'kl/beta': 0.07915578037500381, 'kl/avg_steps': 0.8125, 'epoch': 0.06}
|
|||
|
|
6%|███████▎ | 43/681 [01:51<26:49, 2.52s/it]
6%|███████▍ | 44/681 [01:54<27:27, 2.59s/it]
{'loss': 0.6985, 'grad_norm': 85.54306030273438, 'learning_rate': 3.115942028985507e-07, 'rewards/chosen': -0.02512773871421814, 'rewards/rejected': -1.1819229125976562, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.1567951440811157, 'logps/chosen': -28.09503936767578, 'logps/rejected': -132.7843017578125, 'logps/ref_chosen': -27.785930633544922, 'logps/ref_rejected': -117.58551788330078, 'logits/chosen': -3.3832082748413086, 'logits/rejected': -3.4922008514404297, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.07783857733011246, 'epsilon_dpo/loss_margin_mean': 14.889684677124023, 'epsilon_dpo/beta_margin_mean': 1.1567952632904053, 'epsilon_dpo/beta_margin_std': 0.9610118269920349, 'epsilon_dpo/beta_margin_grad_mean': -0.2732856273651123, 'epsilon_dpo/beta_margin_grad_std': 0.15621285140514374, 'kl/beta': 0.07851782441139221, 'kl/avg_steps': 0.875, 'epoch': 0.06}
|
|||
|
|
6%|███████▍ | 44/681 [01:54<27:27, 2.59s/it]
7%|███████▌ | 45/681 [01:56<26:29, 2.50s/it]
{'loss': 0.8642, 'grad_norm': 86.84064483642578, 'learning_rate': 3.188405797101449e-07, 'rewards/chosen': -0.14034625887870789, 'rewards/rejected': -0.9687749147415161, 'rewards/accuracies': 0.875, 'rewards/margins': 0.8284286260604858, 'logps/chosen': -38.12892150878906, 'logps/rejected': -95.89362335205078, 'logps/ref_chosen': -36.33074951171875, 'logps/ref_rejected': -83.34062194824219, 'logits/chosen': -3.4090023040771484, 'logits/rejected': -3.488529920578003, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.07733368128538132, 'epsilon_dpo/loss_margin_mean': 10.75483226776123, 'epsilon_dpo/beta_margin_mean': 0.8284286260604858, 'epsilon_dpo/beta_margin_std': 0.8817453980445862, 'epsilon_dpo/beta_margin_grad_mean': -0.3332192301750183, 'epsilon_dpo/beta_margin_grad_std': 0.15007154643535614, 'kl/beta': 0.07783675193786621, 'kl/avg_steps': 0.65625, 'epoch': 0.07}
|
|||
|
|
7%|███████▌ | 45/681 [01:56<26:29, 2.50s/it]
7%|███████▊ | 46/681 [01:58<26:23, 2.49s/it]
{'loss': 0.7386, 'grad_norm': 96.92198181152344, 'learning_rate': 3.260869565217391e-07, 'rewards/chosen': -0.13517028093338013, 'rewards/rejected': -1.2355304956436157, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.1003601551055908, 'logps/chosen': -37.75640106201172, 'logps/rejected': -117.52153015136719, 'logps/ref_chosen': -36.0171012878418, 'logps/ref_rejected': -101.40194702148438, 'logits/chosen': -3.375389814376831, 'logits/rejected': -3.439635992050171, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.07675698399543762, 'epsilon_dpo/loss_margin_mean': 14.38028621673584, 'epsilon_dpo/beta_margin_mean': 1.1003601551055908, 'epsilon_dpo/beta_margin_std': 1.0318849086761475, 'epsilon_dpo/beta_margin_grad_mean': -0.2891290783882141, 'epsilon_dpo/beta_margin_grad_std': 0.1580805480480194, 'kl/beta': 0.07732927799224854, 'kl/avg_steps': 0.75, 'epoch': 0.07}
|
|||
|
|
7%|███████▊ | 46/681 [01:58<26:23, 2.49s/it]
7%|███████▉ | 47/681 [02:01<26:19, 2.49s/it]
{'loss': 0.8604, 'grad_norm': 94.45354461669922, 'learning_rate': 3.333333333333333e-07, 'rewards/chosen': -0.19145357608795166, 'rewards/rejected': -1.1218818426132202, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.9304282665252686, 'logps/chosen': -44.316593170166016, 'logps/rejected': -116.04118347167969, 'logps/ref_chosen': -41.82904815673828, 'logps/ref_rejected': -101.29283142089844, 'logits/chosen': -3.4388890266418457, 'logits/rejected': -3.5206193923950195, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.07616160064935684, 'epsilon_dpo/loss_margin_mean': 12.260807037353516, 'epsilon_dpo/beta_margin_mean': 0.9304282069206238, 'epsilon_dpo/beta_margin_std': 1.0573234558105469, 'epsilon_dpo/beta_margin_grad_mean': -0.31451866030693054, 'epsilon_dpo/beta_margin_grad_std': 0.16808247566223145, 'kl/beta': 0.07675362378358841, 'kl/avg_steps': 0.78125, 'epoch': 0.07}
|
|||
|
|
7%|███████▉ | 47/681 [02:01<26:19, 2.49s/it]
7%|████████ | 48/681 [02:04<27:16, 2.59s/it]
{'loss': 0.7607, 'grad_norm': 90.87052154541016, 'learning_rate': 3.4057971014492755e-07, 'rewards/chosen': -0.19789756834506989, 'rewards/rejected': -1.3635492324829102, 'rewards/accuracies': 0.859375, 'rewards/margins': 1.1656516790390015, 'logps/chosen': -38.7716064453125, 'logps/rejected': -113.47309875488281, 'logps/ref_chosen': -36.18339920043945, 'logps/ref_rejected': -95.41502380371094, 'logits/chosen': -3.463271141052246, 'logits/rejected': -3.4467902183532715, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.07571401447057724, 'epsilon_dpo/loss_margin_mean': 15.469880104064941, 'epsilon_dpo/beta_margin_mean': 1.1656516790390015, 'epsilon_dpo/beta_margin_std': 1.2263123989105225, 'epsilon_dpo/beta_margin_grad_mean': -0.29027602076530457, 'epsilon_dpo/beta_margin_grad_std': 0.17632031440734863, 'kl/beta': 0.07615863531827927, 'kl/avg_steps': 0.59375, 'epoch': 0.07}
|
|||
|
|
7%|████████ | 48/681 [02:04<27:16, 2.59s/it]
7%|████████▎ | 49/681 [02:06<26:45, 2.54s/it]
{'loss': 0.6703, 'grad_norm': 81.80361938476562, 'learning_rate': 3.478260869565217e-07, 'rewards/chosen': -0.118332639336586, 'rewards/rejected': -1.4788892269134521, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.3605566024780273, 'logps/chosen': -37.51390075683594, 'logps/rejected': -111.02618408203125, 'logps/ref_chosen': -35.96546936035156, 'logps/ref_rejected': -91.31820678710938, 'logits/chosen': -3.472126007080078, 'logits/rejected': -3.5396313667297363, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.075148805975914, 'epsilon_dpo/loss_margin_mean': 18.15953826904297, 'epsilon_dpo/beta_margin_mean': 1.3605566024780273, 'epsilon_dpo/beta_margin_std': 1.2057913541793823, 'epsilon_dpo/beta_margin_grad_mean': -0.2587027847766876, 'epsilon_dpo/beta_margin_grad_std': 0.17777220904827118, 'kl/beta': 0.07570911198854446, 'kl/avg_steps': 0.75, 'epoch': 0.07}
|
|||
|
|
7%|████████▎ | 49/681 [02:06<26:45, 2.54s/it]
7%|████████▍ | 50/681 [02:09<26:17, 2.50s/it]
{'loss': 0.7726, 'grad_norm': 100.63481903076172, 'learning_rate': 3.5507246376811595e-07, 'rewards/chosen': -0.30415183305740356, 'rewards/rejected': -1.6275885105133057, 'rewards/accuracies': 0.84375, 'rewards/margins': 1.3234367370605469, 'logps/chosen': -46.16408920288086, 'logps/rejected': -122.25482177734375, 'logps/ref_chosen': -42.138206481933594, 'logps/ref_rejected': -100.4173583984375, 'logits/chosen': -3.4349403381347656, 'logits/rejected': -3.533860445022583, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.07468333095312119, 'epsilon_dpo/loss_margin_mean': 17.81157684326172, 'epsilon_dpo/beta_margin_mean': 1.3234366178512573, 'epsilon_dpo/beta_margin_std': 1.4268280267715454, 'epsilon_dpo/beta_margin_grad_mean': -0.27890822291374207, 'epsilon_dpo/beta_margin_grad_std': 0.21263383328914642, 'kl/beta': 0.07514552026987076, 'kl/avg_steps': 0.625, 'epoch': 0.07}
|
|||
|
|
7%|████████▍ | 50/681 [02:09<26:17, 2.50s/it]
7%|████████▌ | 51/681 [02:11<26:21, 2.51s/it]
{'loss': 0.7605, 'grad_norm': 91.88065338134766, 'learning_rate': 3.6231884057971015e-07, 'rewards/chosen': -0.3597980737686157, 'rewards/rejected': -1.7088831663131714, 'rewards/accuracies': 0.875, 'rewards/margins': 1.3490850925445557, 'logps/chosen': -43.83557891845703, 'logps/rejected': -103.68801879882812, 'logps/ref_chosen': -39.016597747802734, 'logps/ref_rejected': -80.60652160644531, 'logits/chosen': -3.377105236053467, 'logits/rejected': -3.4192728996276855, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.07421945780515671, 'epsilon_dpo/loss_margin_mean': 18.262516021728516, 'epsilon_dpo/beta_margin_mean': 1.3490850925445557, 'epsilon_dpo/beta_margin_std': 1.5637013912200928, 'epsilon_dpo/beta_margin_grad_mean': -0.28537219762802124, 'epsilon_dpo/beta_margin_grad_std': 0.19426609575748444, 'kl/beta': 0.07467877864837646, 'kl/avg_steps': 0.625, 'epoch': 0.07}
|
|||
|
|
7%|████████▌ | 51/681 [02:11<26:21, 2.51s/it]
8%|████████▊ | 52/681 [02:14<26:30, 2.53s/it]
{'loss': 0.6001, 'grad_norm': 87.76582336425781, 'learning_rate': 3.695652173913043e-07, 'rewards/chosen': -0.3565700650215149, 'rewards/rejected': -2.0638861656188965, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.7073161602020264, 'logps/chosen': -39.097511291503906, 'logps/rejected': -114.01925659179688, 'logps/ref_chosen': -34.285945892333984, 'logps/ref_rejected': -85.96109008789062, 'logits/chosen': -3.4000444412231445, 'logits/rejected': -3.473278284072876, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.07364249974489212, 'epsilon_dpo/loss_margin_mean': 23.246597290039062, 'epsilon_dpo/beta_margin_mean': 1.7073160409927368, 'epsilon_dpo/beta_margin_std': 1.5912011861801147, 'epsilon_dpo/beta_margin_grad_mean': -0.23005272448062897, 'epsilon_dpo/beta_margin_grad_std': 0.1819276660680771, 'kl/beta': 0.07421493530273438, 'kl/avg_steps': 0.78125, 'epoch': 0.08}
|
|||
|
|
8%|████████▊ | 52/681 [02:14<26:30, 2.53s/it]
8%|████████▉ | 53/681 [02:16<26:43, 2.55s/it]
{'loss': 0.558, 'grad_norm': 85.12645721435547, 'learning_rate': 3.7681159420289855e-07, 'rewards/chosen': -0.5423527359962463, 'rewards/rejected': -2.3573391437530518, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.8149864673614502, 'logps/chosen': -42.113983154296875, 'logps/rejected': -129.96218872070312, 'logps/ref_chosen': -34.706817626953125, 'logps/ref_rejected': -97.64952087402344, 'logits/chosen': -3.3786673545837402, 'logits/rejected': -3.4282350540161133, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.07307162135839462, 'epsilon_dpo/loss_margin_mean': 24.905494689941406, 'epsilon_dpo/beta_margin_mean': 1.8149864673614502, 'epsilon_dpo/beta_margin_std': 1.6574300527572632, 'epsilon_dpo/beta_margin_grad_mean': -0.21757791936397552, 'epsilon_dpo/beta_margin_grad_std': 0.17687389254570007, 'kl/beta': 0.07363962382078171, 'kl/avg_steps': 0.78125, 'epoch': 0.08}
|
|||
|
|
8%|████████▉ | 53/681 [02:16<26:43, 2.55s/it]
8%|█████████ | 54/681 [02:19<26:08, 2.50s/it]
{'loss': 0.6017, 'grad_norm': 83.51258850097656, 'learning_rate': 3.8405797101449274e-07, 'rewards/chosen': -0.46132320165634155, 'rewards/rejected': -2.206540822982788, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.7452175617218018, 'logps/chosen': -46.09777069091797, 'logps/rejected': -129.1588592529297, 'logps/ref_chosen': -39.777854919433594, 'logps/ref_rejected': -98.70614624023438, 'logits/chosen': -3.400343656539917, 'logits/rejected': -3.5012292861938477, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.07252801209688187, 'epsilon_dpo/loss_margin_mean': 24.13280487060547, 'epsilon_dpo/beta_margin_mean': 1.7452175617218018, 'epsilon_dpo/beta_margin_std': 1.7455625534057617, 'epsilon_dpo/beta_margin_grad_mean': -0.23283910751342773, 'epsilon_dpo/beta_margin_grad_std': 0.18114687502384186, 'kl/beta': 0.0730687752366066, 'kl/avg_steps': 0.75, 'epoch': 0.08}
|
|||
|
|
8%|█████████ | 54/681 [02:19<26:08, 2.50s/it]
8%|█████████▎ | 55/681 [02:21<25:34, 2.45s/it]
{'loss': 0.7261, 'grad_norm': 104.0904541015625, 'learning_rate': 3.9130434782608694e-07, 'rewards/chosen': -0.7076321840286255, 'rewards/rejected': -2.3510682582855225, 'rewards/accuracies': 0.796875, 'rewards/margins': 1.643436074256897, 'logps/chosen': -58.18317794799805, 'logps/rejected': -126.38877868652344, 'logps/ref_chosen': -48.42914962768555, 'logps/ref_rejected': -93.72831726074219, 'logits/chosen': -3.390665054321289, 'logits/rejected': -3.4186453819274902, 'kl/p_epsilon_steps': 0.765625, 'kl/n_epsilon_steps': 0.234375, 'epsilon_dpo/beta': 0.07214676588773727, 'epsilon_dpo/loss_margin_mean': 22.906444549560547, 'epsilon_dpo/beta_margin_mean': 1.643436074256897, 'epsilon_dpo/beta_margin_std': 1.8868768215179443, 'epsilon_dpo/beta_margin_grad_mean': -0.2661394476890564, 'epsilon_dpo/beta_margin_grad_std': 0.21455174684524536, 'kl/beta': 0.07252483814954758, 'kl/avg_steps': 0.53125, 'epoch': 0.08}
|
|||
|
|
8%|█████████▎ | 55/681 [02:21<25:34, 2.45s/it]
8%|█████████▍ | 56/681 [02:24<26:06, 2.51s/it]
{'loss': 0.713, 'grad_norm': 123.8470458984375, 'learning_rate': 3.9855072463768114e-07, 'rewards/chosen': -0.5154626369476318, 'rewards/rejected': -2.4932093620300293, 'rewards/accuracies': 0.84375, 'rewards/margins': 1.9777467250823975, 'logps/chosen': -38.82612991333008, 'logps/rejected': -135.484375, 'logps/ref_chosen': -31.692344665527344, 'logps/ref_rejected': -100.61968994140625, 'logits/chosen': -3.2844839096069336, 'logits/rejected': -3.3951363563537598, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.07168648391962051, 'epsilon_dpo/loss_margin_mean': 27.73088836669922, 'epsilon_dpo/beta_margin_mean': 1.977746605873108, 'epsilon_dpo/beta_margin_std': 2.0250532627105713, 'epsilon_dpo/beta_margin_grad_mean': -0.2267267107963562, 'epsilon_dpo/beta_margin_grad_std': 0.24484246969223022, 'kl/beta': 0.07214158773422241, 'kl/avg_steps': 0.640625, 'epoch': 0.08}
|
|||
|
|
8%|█████████▍ | 56/681 [02:24<26:06, 2.51s/it]
8%|█████████▋ | 57/681 [02:26<26:18, 2.53s/it]
{'loss': 0.6021, 'grad_norm': 102.96115112304688, 'learning_rate': 4.057971014492754e-07, 'rewards/chosen': -0.5308903455734253, 'rewards/rejected': -2.707615375518799, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.176724910736084, 'logps/chosen': -45.702720642089844, 'logps/rejected': -139.84365844726562, 'logps/ref_chosen': -38.302345275878906, 'logps/ref_rejected': -101.74482727050781, 'logits/chosen': -3.386384963989258, 'logits/rejected': -3.439547538757324, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.07115186750888824, 'epsilon_dpo/loss_margin_mean': 30.698436737060547, 'epsilon_dpo/beta_margin_mean': 2.176724910736084, 'epsilon_dpo/beta_margin_std': 2.1261887550354004, 'epsilon_dpo/beta_margin_grad_mean': -0.21097464859485626, 'epsilon_dpo/beta_margin_grad_std': 0.22179222106933594, 'kl/beta': 0.07168237119913101, 'kl/avg_steps': 0.75, 'epoch': 0.08}
|
|||
|
|
8%|█████████▋ | 57/681 [02:26<26:18, 2.53s/it]
9%|█████████▊ | 58/681 [02:29<25:52, 2.49s/it]
{'loss': 0.4979, 'grad_norm': 87.39006805419922, 'learning_rate': 4.1304347826086954e-07, 'rewards/chosen': -0.5473577976226807, 'rewards/rejected': -2.704098701477051, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.15674090385437, 'logps/chosen': -46.135406494140625, 'logps/rejected': -127.86618041992188, 'logps/ref_chosen': -38.44845962524414, 'logps/ref_rejected': -89.55912780761719, 'logits/chosen': -3.3047280311584473, 'logits/rejected': -3.393859386444092, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.07062220573425293, 'epsilon_dpo/loss_margin_mean': 30.6201114654541, 'epsilon_dpo/beta_margin_mean': 2.15674090385437, 'epsilon_dpo/beta_margin_std': 2.0642378330230713, 'epsilon_dpo/beta_margin_grad_mean': -0.1960882991552353, 'epsilon_dpo/beta_margin_grad_std': 0.17183293402194977, 'kl/beta': 0.07114876061677933, 'kl/avg_steps': 0.75, 'epoch': 0.09}
|
|||
|
|
9%|█████████▊ | 58/681 [02:29<25:52, 2.49s/it]
9%|█████████▉ | 59/681 [02:31<25:37, 2.47s/it]
{'loss': 0.6172, 'grad_norm': 87.90422058105469, 'learning_rate': 4.2028985507246374e-07, 'rewards/chosen': -0.5922181010246277, 'rewards/rejected': -2.7694854736328125, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.177267074584961, 'logps/chosen': -46.44553756713867, 'logps/rejected': -133.6805419921875, 'logps/ref_chosen': -38.029998779296875, 'logps/ref_rejected': -94.10072326660156, 'logits/chosen': -3.3299951553344727, 'logits/rejected': -3.408975124359131, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0700632631778717, 'epsilon_dpo/loss_margin_mean': 31.16427993774414, 'epsilon_dpo/beta_margin_mean': 2.177267074584961, 'epsilon_dpo/beta_margin_std': 2.287863254547119, 'epsilon_dpo/beta_margin_grad_mean': -0.21717199683189392, 'epsilon_dpo/beta_margin_grad_std': 0.2148067206144333, 'kl/beta': 0.0706191137433052, 'kl/avg_steps': 0.796875, 'epoch': 0.09}
|
|||
|
|
9%|█████████▉ | 59/681 [02:31<25:37, 2.47s/it]
9%|██████████▏ | 60/681 [02:34<26:17, 2.54s/it]
{'loss': 0.9335, 'grad_norm': 167.74856567382812, 'learning_rate': 4.2753623188405794e-07, 'rewards/chosen': -0.9826135635375977, 'rewards/rejected': -2.746914863586426, 'rewards/accuracies': 0.828125, 'rewards/margins': 1.7643013000488281, 'logps/chosen': -62.78964614868164, 'logps/rejected': -130.8575439453125, 'logps/ref_chosen': -48.789947509765625, 'logps/ref_rejected': -91.3543701171875, 'logits/chosen': -3.3077573776245117, 'logits/rejected': -3.2960987091064453, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.06962990015745163, 'epsilon_dpo/loss_margin_mean': 25.50347328186035, 'epsilon_dpo/beta_margin_mean': 1.7643014192581177, 'epsilon_dpo/beta_margin_std': 2.2814714908599854, 'epsilon_dpo/beta_margin_grad_mean': -0.24627114832401276, 'epsilon_dpo/beta_margin_grad_std': 0.2656000852584839, 'kl/beta': 0.07006081938743591, 'kl/avg_steps': 0.625, 'epoch': 0.09}
|
|||
|
|
9%|██████████▏ | 60/681 [02:34<26:17, 2.54s/it]
9%|██████████▎ | 61/681 [02:36<26:18, 2.55s/it]
{'loss': 0.6385, 'grad_norm': 96.6867446899414, 'learning_rate': 4.3478260869565214e-07, 'rewards/chosen': -0.7702499628067017, 'rewards/rejected': -2.9774599075317383, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.207209825515747, 'logps/chosen': -47.05156707763672, 'logps/rejected': -138.587158203125, 'logps/ref_chosen': -35.972103118896484, 'logps/ref_rejected': -95.45098876953125, 'logits/chosen': -3.313138484954834, 'logits/rejected': -3.4350461959838867, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.06911037117242813, 'epsilon_dpo/loss_margin_mean': 32.056705474853516, 'epsilon_dpo/beta_margin_mean': 2.207209825515747, 'epsilon_dpo/beta_margin_std': 2.3618977069854736, 'epsilon_dpo/beta_margin_grad_mean': -0.20471161603927612, 'epsilon_dpo/beta_margin_grad_std': 0.22111034393310547, 'kl/beta': 0.06962565332651138, 'kl/avg_steps': 0.75, 'epoch': 0.09}
|
|||
|
|
9%|██████████▎ | 61/681 [02:36<26:18, 2.55s/it]
9%|██████████▍ | 62/681 [02:39<26:16, 2.55s/it]
{'loss': 0.6249, 'grad_norm': 91.73394775390625, 'learning_rate': 4.420289855072464e-07, 'rewards/chosen': -0.6530295610427856, 'rewards/rejected': -2.504047155380249, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.8510175943374634, 'logps/chosen': -45.381500244140625, 'logps/rejected': -119.45492553710938, 'logps/ref_chosen': -35.904327392578125, 'logps/ref_rejected': -82.9093017578125, 'logits/chosen': -3.2718987464904785, 'logits/rejected': -3.3174455165863037, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0686391070485115, 'epsilon_dpo/loss_margin_mean': 27.06845474243164, 'epsilon_dpo/beta_margin_mean': 1.8510175943374634, 'epsilon_dpo/beta_margin_std': 1.8581922054290771, 'epsilon_dpo/beta_margin_grad_mean': -0.2324148714542389, 'epsilon_dpo/beta_margin_grad_std': 0.20171168446540833, 'kl/beta': 0.06910735368728638, 'kl/avg_steps': 0.6875, 'epoch': 0.09}
|
|||
|
|
9%|██████████▍ | 62/681 [02:39<26:16, 2.55s/it]
9%|██████████▋ | 63/681 [02:41<25:58, 2.52s/it]
{'loss': 0.6897, 'grad_norm': 102.05279541015625, 'learning_rate': 4.4927536231884053e-07, 'rewards/chosen': -0.9690430164337158, 'rewards/rejected': -3.1313982009887695, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.1623551845550537, 'logps/chosen': -60.41229248046875, 'logps/rejected': -150.15708923339844, 'logps/ref_chosen': -46.25957107543945, 'logps/ref_rejected': -104.15571594238281, 'logits/chosen': -3.3514161109924316, 'logits/rejected': -3.4100210666656494, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.06817042827606201, 'epsilon_dpo/loss_margin_mean': 31.848644256591797, 'epsilon_dpo/beta_margin_mean': 2.1623551845550537, 'epsilon_dpo/beta_margin_std': 2.539600372314453, 'epsilon_dpo/beta_margin_grad_mean': -0.22793808579444885, 'epsilon_dpo/beta_margin_grad_std': 0.2142634242773056, 'kl/beta': 0.0686354786157608, 'kl/avg_steps': 0.6875, 'epoch': 0.09}
|
|||
|
|
9%|██████████▋ | 63/681 [02:41<25:58, 2.52s/it]
9%|██████████▊ | 64/681 [02:44<25:37, 2.49s/it]
{'loss': 0.5255, 'grad_norm': 95.53367614746094, 'learning_rate': 4.5652173913043473e-07, 'rewards/chosen': -0.6991441249847412, 'rewards/rejected': -3.055689811706543, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.3565456867218018, 'logps/chosen': -44.803466796875, 'logps/rejected': -146.42654418945312, 'logps/ref_chosen': -34.512210845947266, 'logps/ref_rejected': -101.21166229248047, 'logits/chosen': -3.3310346603393555, 'logits/rejected': -3.3657891750335693, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.06770496070384979, 'epsilon_dpo/loss_margin_mean': 34.92362594604492, 'epsilon_dpo/beta_margin_mean': 2.3565456867218018, 'epsilon_dpo/beta_margin_std': 2.4017789363861084, 'epsilon_dpo/beta_margin_grad_mean': -0.19847899675369263, 'epsilon_dpo/beta_margin_grad_std': 0.19424547255039215, 'kl/beta': 0.0681668370962143, 'kl/avg_steps': 0.6875, 'epoch': 0.09}
|
|||
|
|
9%|██████████▊ | 64/681 [02:44<25:37, 2.49s/it]
10%|██████████▉ | 65/681 [02:46<25:44, 2.51s/it]
{'loss': 0.5986, 'grad_norm': 93.06818389892578, 'learning_rate': 4.63768115942029e-07, 'rewards/chosen': -0.6990154981613159, 'rewards/rejected': -3.1816627979278564, 'rewards/accuracies': 0.859375, 'rewards/margins': 2.48264741897583, 'logps/chosen': -53.97098159790039, 'logps/rejected': -162.62474060058594, 'logps/ref_chosen': -43.620361328125, 'logps/ref_rejected': -115.21531677246094, 'logits/chosen': -3.3164443969726562, 'logits/rejected': -3.343629837036133, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.06730613857507706, 'epsilon_dpo/loss_margin_mean': 37.058799743652344, 'epsilon_dpo/beta_margin_mean': 2.48264741897583, 'epsilon_dpo/beta_margin_std': 2.5546720027923584, 'epsilon_dpo/beta_margin_grad_mean': -0.21261604130268097, 'epsilon_dpo/beta_margin_grad_std': 0.22306111454963684, 'kl/beta': 0.06770138442516327, 'kl/avg_steps': 0.59375, 'epoch': 0.1}
|
|||
|
|
10%|██████████▉ | 65/681 [02:46<25:44, 2.51s/it]
10%|███████████▏ | 66/681 [02:49<25:51, 2.52s/it]
{'loss': 0.7582, 'grad_norm': 116.8039321899414, 'learning_rate': 4.7101449275362313e-07, 'rewards/chosen': -0.7809337973594666, 'rewards/rejected': -2.8556485176086426, 'rewards/accuracies': 0.796875, 'rewards/margins': 2.0747146606445312, 'logps/chosen': -49.09389877319336, 'logps/rejected': -123.10272216796875, 'logps/ref_chosen': -37.514625549316406, 'logps/ref_rejected': -80.34272766113281, 'logits/chosen': -3.2685537338256836, 'logits/rejected': -3.3573737144470215, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.0669299066066742, 'epsilon_dpo/loss_margin_mean': 31.18071746826172, 'epsilon_dpo/beta_margin_mean': 2.0747146606445312, 'epsilon_dpo/beta_margin_std': 2.3843464851379395, 'epsilon_dpo/beta_margin_grad_mean': -0.24468040466308594, 'epsilon_dpo/beta_margin_grad_std': 0.2503945231437683, 'kl/beta': 0.06730178743600845, 'kl/avg_steps': 0.5625, 'epoch': 0.1}
|
|||
|
|
10%|███████████▏ | 66/681 [02:49<25:51, 2.52s/it]
10%|███████████▎ | 67/681 [02:51<25:21, 2.48s/it]
{'loss': 0.7213, 'grad_norm': 115.71697998046875, 'learning_rate': 4.782608695652174e-07, 'rewards/chosen': -0.8721305727958679, 'rewards/rejected': -2.741394519805908, 'rewards/accuracies': 0.84375, 'rewards/margins': 1.8692641258239746, 'logps/chosen': -51.816200256347656, 'logps/rejected': -119.65333557128906, 'logps/ref_chosen': -38.82200622558594, 'logps/ref_rejected': -78.41658782958984, 'logits/chosen': -3.2035374641418457, 'logits/rejected': -3.216823101043701, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0665346160531044, 'epsilon_dpo/loss_margin_mean': 28.2425479888916, 'epsilon_dpo/beta_margin_mean': 1.869264006614685, 'epsilon_dpo/beta_margin_std': 2.120335340499878, 'epsilon_dpo/beta_margin_grad_mean': -0.2480592578649521, 'epsilon_dpo/beta_margin_grad_std': 0.23476538062095642, 'kl/beta': 0.06692533195018768, 'kl/avg_steps': 0.59375, 'epoch': 0.1}
|
|||
|
|
10%|███████████▎ | 67/681 [02:51<25:21, 2.48s/it]
10%|███████████▍ | 68/681 [02:54<25:34, 2.50s/it]
{'loss': 0.5268, 'grad_norm': 89.41032409667969, 'learning_rate': 4.855072463768116e-07, 'rewards/chosen': -0.7523297071456909, 'rewards/rejected': -2.9893717765808105, 'rewards/accuracies': 0.875, 'rewards/margins': 2.237042188644409, 'logps/chosen': -53.25446319580078, 'logps/rejected': -127.92015075683594, 'logps/ref_chosen': -41.910316467285156, 'logps/ref_rejected': -82.59764862060547, 'logits/chosen': -3.2750229835510254, 'logits/rejected': -3.2742607593536377, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.06605872511863708, 'epsilon_dpo/loss_margin_mean': 33.978355407714844, 'epsilon_dpo/beta_margin_mean': 2.237042188644409, 'epsilon_dpo/beta_margin_std': 1.8940068483352661, 'epsilon_dpo/beta_margin_grad_mean': -0.19077911972999573, 'epsilon_dpo/beta_margin_grad_std': 0.20632153749465942, 'kl/beta': 0.06653030216693878, 'kl/avg_steps': 0.71875, 'epoch': 0.1}
|
|||
|
|
10%|███████████▍ | 68/681 [02:54<25:34, 2.50s/it]
10%|███████████▋ | 69/681 [02:56<25:59, 2.55s/it]
{'loss': 0.5513, 'grad_norm': 85.71179962158203, 'learning_rate': 4.927536231884058e-07, 'rewards/chosen': -0.910045325756073, 'rewards/rejected': -3.3686389923095703, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.4585933685302734, 'logps/chosen': -56.83376693725586, 'logps/rejected': -162.74642944335938, 'logps/ref_chosen': -42.98963165283203, 'logps/ref_rejected': -111.28137969970703, 'logits/chosen': -3.309999704360962, 'logits/rejected': -3.2920103073120117, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.06550473719835281, 'epsilon_dpo/loss_margin_mean': 37.620914459228516, 'epsilon_dpo/beta_margin_mean': 2.4585933685302734, 'epsilon_dpo/beta_margin_std': 2.252986431121826, 'epsilon_dpo/beta_margin_grad_mean': -0.1768295019865036, 'epsilon_dpo/beta_margin_grad_std': 0.19139200448989868, 'kl/beta': 0.066055528819561, 'kl/avg_steps': 0.84375, 'epoch': 0.1}
|
|||
|
|
10%|███████████▋ | 69/681 [02:56<25:59, 2.55s/it]
10%|███████████▊ | 70/681 [02:59<25:41, 2.52s/it]
{'loss': 0.5016, 'grad_norm': 88.73102569580078, 'learning_rate': 5e-07, 'rewards/chosen': -0.781528651714325, 'rewards/rejected': -3.0996618270874023, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.3181333541870117, 'logps/chosen': -55.655487060546875, 'logps/rejected': -144.90789794921875, 'logps/ref_chosen': -43.68109130859375, 'logps/ref_rejected': -97.17718505859375, 'logits/chosen': -3.2341670989990234, 'logits/rejected': -3.3183038234710693, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.06497713923454285, 'epsilon_dpo/loss_margin_mean': 35.75630187988281, 'epsilon_dpo/beta_margin_mean': 2.3181331157684326, 'epsilon_dpo/beta_margin_std': 2.1517505645751953, 'epsilon_dpo/beta_margin_grad_mean': -0.1917823851108551, 'epsilon_dpo/beta_margin_grad_std': 0.18596571683883667, 'kl/beta': 0.06550285220146179, 'kl/avg_steps': 0.8125, 'epoch': 0.1}
|
|||
|
|
10%|███████████▊ | 70/681 [02:59<25:41, 2.52s/it]
10%|███████████▉ | 71/681 [03:01<25:55, 2.55s/it]
{'loss': 0.5652, 'grad_norm': 100.56282806396484, 'learning_rate': 4.999967061337492e-07, 'rewards/chosen': -0.8535667657852173, 'rewards/rejected': -3.5020976066589355, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.6485307216644287, 'logps/chosen': -54.0555419921875, 'logps/rejected': -158.8681182861328, 'logps/ref_chosen': -40.898582458496094, 'logps/ref_rejected': -104.50498962402344, 'logits/chosen': -3.228590488433838, 'logits/rejected': -3.3432466983795166, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0644940659403801, 'epsilon_dpo/loss_margin_mean': 41.206172943115234, 'epsilon_dpo/beta_margin_mean': 2.6485307216644287, 'epsilon_dpo/beta_margin_std': 2.504869222640991, 'epsilon_dpo/beta_margin_grad_mean': -0.18719229102134705, 'epsilon_dpo/beta_margin_grad_std': 0.23163102567195892, 'kl/beta': 0.06497492641210556, 'kl/avg_steps': 0.75, 'epoch': 0.1}
|
|||
|
|
10%|███████████▉ | 71/681 [03:02<25:55, 2.55s/it]
11%|████████████▏ | 72/681 [03:04<25:42, 2.53s/it]
{'loss': 0.4074, 'grad_norm': 71.43647766113281, 'learning_rate': 4.999868246217933e-07, 'rewards/chosen': -0.8116366863250732, 'rewards/rejected': -3.40474271774292, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.593106269836426, 'logps/chosen': -54.79957580566406, 'logps/rejected': -155.27841186523438, 'logps/ref_chosen': -42.15618896484375, 'logps/ref_rejected': -102.02656555175781, 'logits/chosen': -3.2843093872070312, 'logits/rejected': -3.316100835800171, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.06397364288568497, 'epsilon_dpo/loss_margin_mean': 40.60845184326172, 'epsilon_dpo/beta_margin_mean': 2.5931060314178467, 'epsilon_dpo/beta_margin_std': 2.0323598384857178, 'epsilon_dpo/beta_margin_grad_mean': -0.15650521218776703, 'epsilon_dpo/beta_margin_grad_std': 0.17222045361995697, 'kl/beta': 0.06449124217033386, 'kl/avg_steps': 0.8125, 'epoch': 0.11}
|
|||
|
|
11%|████████████▏ | 72/681 [03:04<25:42, 2.53s/it]
11%|████████████▎ | 73/681 [03:07<26:25, 2.61s/it]
{'loss': 0.5577, 'grad_norm': 75.35091400146484, 'learning_rate': 4.999703557245192e-07, 'rewards/chosen': -0.7197288870811462, 'rewards/rejected': -3.2321105003356934, 'rewards/accuracies': 0.859375, 'rewards/margins': 2.5123815536499023, 'logps/chosen': -55.1505126953125, 'logps/rejected': -147.10968017578125, 'logps/ref_chosen': -43.86912155151367, 'logps/ref_rejected': -96.146728515625, 'logits/chosen': -3.2197837829589844, 'logits/rejected': -3.2296009063720703, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.06355801969766617, 'epsilon_dpo/loss_margin_mean': 39.681556701660156, 'epsilon_dpo/beta_margin_mean': 2.5123815536499023, 'epsilon_dpo/beta_margin_std': 2.4150218963623047, 'epsilon_dpo/beta_margin_grad_mean': -0.2032501995563507, 'epsilon_dpo/beta_margin_grad_std': 0.21837185323238373, 'kl/beta': 0.06397147476673126, 'kl/avg_steps': 0.65625, 'epoch': 0.11}
|
|||
|
|
11%|████████████▎ | 73/681 [03:07<26:25, 2.61s/it]
11%|████████████▍ | 74/681 [03:09<25:47, 2.55s/it]
{'loss': 0.4061, 'grad_norm': 65.50677490234375, 'learning_rate': 4.999472998758977e-07, 'rewards/chosen': -0.6169643998146057, 'rewards/rejected': -3.4588942527770996, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.8419299125671387, 'logps/chosen': -38.77723693847656, 'logps/rejected': -157.6680145263672, 'logps/ref_chosen': -29.008399963378906, 'logps/ref_rejected': -102.72833251953125, 'logits/chosen': -3.147824764251709, 'logits/rejected': -3.291006088256836, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.06304432451725006, 'epsilon_dpo/loss_margin_mean': 45.170841217041016, 'epsilon_dpo/beta_margin_mean': 2.8419299125671387, 'epsilon_dpo/beta_margin_std': 2.680656909942627, 'epsilon_dpo/beta_margin_grad_mean': -0.15927036106586456, 'epsilon_dpo/beta_margin_grad_std': 0.18017472326755524, 'kl/beta': 0.06355439871549606, 'kl/avg_steps': 0.8125, 'epoch': 0.11}
|
|||
|
|
11%|████████████▍ | 74/681 [03:09<25:47, 2.55s/it]
11%|████████████▋ | 75/681 [03:12<25:24, 2.52s/it]
{'loss': 0.3702, 'grad_norm': 62.55946731567383, 'learning_rate': 4.999176576834721e-07, 'rewards/chosen': -0.9213298559188843, 'rewards/rejected': -3.7345452308654785, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.8132152557373047, 'logps/chosen': -45.443458557128906, 'logps/rejected': -177.2479705810547, 'logps/ref_chosen': -30.710708618164062, 'logps/ref_rejected': -117.44107818603516, 'logits/chosen': -3.151859760284424, 'logits/rejected': -3.281860828399658, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.062496818602085114, 'epsilon_dpo/loss_margin_mean': 45.07414245605469, 'epsilon_dpo/beta_margin_mean': 2.8132150173187256, 'epsilon_dpo/beta_margin_std': 2.3551828861236572, 'epsilon_dpo/beta_margin_grad_mean': -0.15049783885478973, 'epsilon_dpo/beta_margin_grad_std': 0.1585647463798523, 'kl/beta': 0.06304218620061874, 'kl/avg_steps': 0.875, 'epoch': 0.11}
|
|||
|
|
11%|████████████▋ | 75/681 [03:12<25:24, 2.52s/it]
11%|████████████▊ | 76/681 [03:14<25:23, 2.52s/it]
{'loss': 0.5828, 'grad_norm': 96.22077941894531, 'learning_rate': 4.998814299283415e-07, 'rewards/chosen': -0.8483954668045044, 'rewards/rejected': -3.050872802734375, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.20247745513916, 'logps/chosen': -48.6707763671875, 'logps/rejected': -134.10556030273438, 'logps/ref_chosen': -35.03684997558594, 'logps/ref_rejected': -84.84458923339844, 'logits/chosen': -3.207365036010742, 'logits/rejected': -3.260375499725342, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.062013305723667145, 'epsilon_dpo/loss_margin_mean': 35.627044677734375, 'epsilon_dpo/beta_margin_mean': 2.20247745513916, 'epsilon_dpo/beta_margin_std': 1.9251240491867065, 'epsilon_dpo/beta_margin_grad_mean': -0.19430118799209595, 'epsilon_dpo/beta_margin_grad_std': 0.2106785923242569, 'kl/beta': 0.06249535083770752, 'kl/avg_steps': 0.78125, 'epoch': 0.11}
|
|||
|
|
11%|████████████▊ | 76/681 [03:14<25:23, 2.52s/it]
11%|█████████████ | 77/681 [03:16<24:51, 2.47s/it]
{'loss': 0.5577, 'grad_norm': 109.90064239501953, 'learning_rate': 4.998386175651409e-07, 'rewards/chosen': -0.8574961423873901, 'rewards/rejected': -3.405547618865967, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.548051357269287, 'logps/chosen': -57.969879150390625, 'logps/rejected': -154.14987182617188, 'logps/ref_chosen': -44.09752655029297, 'logps/ref_rejected': -98.75190734863281, 'logits/chosen': -3.159731864929199, 'logits/rejected': -3.2670416831970215, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.06151320040225983, 'epsilon_dpo/loss_margin_mean': 41.525611877441406, 'epsilon_dpo/beta_margin_mean': 2.548051595687866, 'epsilon_dpo/beta_margin_std': 2.222637414932251, 'epsilon_dpo/beta_margin_grad_mean': -0.16589321196079254, 'epsilon_dpo/beta_margin_grad_std': 0.19776158034801483, 'kl/beta': 0.06201088801026344, 'kl/avg_steps': 0.8125, 'epoch': 0.11}
|
|||
|
|
11%|█████████████ | 77/681 [03:17<24:51, 2.47s/it]
11%|█████████████▏ | 78/681 [03:19<25:11, 2.51s/it]
{'loss': 0.5417, 'grad_norm': 88.5641860961914, 'learning_rate': 4.997892217220159e-07, 'rewards/chosen': -0.8202856779098511, 'rewards/rejected': -2.9441146850585938, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.123828887939453, 'logps/chosen': -52.061004638671875, 'logps/rejected': -139.2445068359375, 'logps/ref_chosen': -38.710906982421875, 'logps/ref_rejected': -91.00759887695312, 'logits/chosen': -3.2637405395507812, 'logits/rejected': -3.250288963317871, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.06109432876110077, 'epsilon_dpo/loss_margin_mean': 34.886810302734375, 'epsilon_dpo/beta_margin_mean': 2.1238291263580322, 'epsilon_dpo/beta_margin_std': 1.9207454919815063, 'epsilon_dpo/beta_margin_grad_mean': -0.19816353917121887, 'epsilon_dpo/beta_margin_grad_std': 0.19675467908382416, 'kl/beta': 0.06151111051440239, 'kl/avg_steps': 0.6875, 'epoch': 0.11}
|
|||
|
|
11%|█████████████▏ | 78/681 [03:19<25:11, 2.51s/it]
12%|█████████████▎ | 79/681 [03:22<25:17, 2.52s/it]
{'loss': 0.4274, 'grad_norm': 86.2493667602539, 'learning_rate': 4.997332437005931e-07, 'rewards/chosen': -0.6022886633872986, 'rewards/rejected': -3.202016592025757, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.5997278690338135, 'logps/chosen': -42.79743957519531, 'logps/rejected': -148.58889770507812, 'logps/ref_chosen': -32.905845642089844, 'logps/ref_rejected': -95.70394897460938, 'logits/chosen': -3.1998233795166016, 'logits/rejected': -3.2696633338928223, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.060581713914871216, 'epsilon_dpo/loss_margin_mean': 42.99335479736328, 'epsilon_dpo/beta_margin_mean': 2.5997278690338135, 'epsilon_dpo/beta_margin_std': 2.096586227416992, 'epsilon_dpo/beta_margin_grad_mean': -0.1641714721918106, 'epsilon_dpo/beta_margin_grad_std': 0.17812295258045197, 'kl/beta': 0.0610911101102829, 'kl/avg_steps': 0.84375, 'epoch': 0.12}
|
|||
|
|
12%|█████████████▎ | 79/681 [03:22<25:17, 2.52s/it]
12%|█████████████▌ | 80/681 [03:24<25:14, 2.52s/it]
{'loss': 0.6202, 'grad_norm': 94.2277603149414, 'learning_rate': 4.996706849759452e-07, 'rewards/chosen': -0.8336386680603027, 'rewards/rejected': -3.2143394947052, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.3807008266448975, 'logps/chosen': -55.867835998535156, 'logps/rejected': -147.4423828125, 'logps/ref_chosen': -42.08654022216797, 'logps/ref_rejected': -93.93815612792969, 'logits/chosen': -3.2733001708984375, 'logits/rejected': -3.3127360343933105, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.06015056371688843, 'epsilon_dpo/loss_margin_mean': 39.722938537597656, 'epsilon_dpo/beta_margin_mean': 2.3807008266448975, 'epsilon_dpo/beta_margin_std': 2.437171220779419, 'epsilon_dpo/beta_margin_grad_mean': -0.20884603261947632, 'epsilon_dpo/beta_margin_grad_std': 0.22885563969612122, 'kl/beta': 0.06057996675372124, 'kl/avg_steps': 0.71875, 'epoch': 0.12}
|
|||
|
|
12%|█████████████▌ | 80/681 [03:24<25:14, 2.52s/it]
12%|█████████████▋ | 81/681 [03:27<25:59, 2.60s/it]
{'loss': 0.4684, 'grad_norm': 70.84126281738281, 'learning_rate': 4.996015471965529e-07, 'rewards/chosen': -0.7309558391571045, 'rewards/rejected': -3.3320164680480957, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.601060628890991, 'logps/chosen': -52.02825927734375, 'logps/rejected': -188.8577117919922, 'logps/ref_chosen': -39.808433532714844, 'logps/ref_rejected': -132.9473876953125, 'logits/chosen': -3.1918153762817383, 'logits/rejected': -3.2612764835357666, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.05966491997241974, 'epsilon_dpo/loss_margin_mean': 43.69050598144531, 'epsilon_dpo/beta_margin_mean': 2.601060628890991, 'epsilon_dpo/beta_margin_std': 2.4161243438720703, 'epsilon_dpo/beta_margin_grad_mean': -0.17548146843910217, 'epsilon_dpo/beta_margin_grad_std': 0.19263611733913422, 'kl/beta': 0.06014765426516533, 'kl/avg_steps': 0.8125, 'epoch': 0.12}
|
|||
|
|
12%|█████████████▋ | 81/681 [03:27<25:59, 2.60s/it]
12%|█████████████▊ | 82/681 [03:29<25:42, 2.57s/it]
{'loss': 0.6066, 'grad_norm': 98.73262023925781, 'learning_rate': 4.995258321842611e-07, 'rewards/chosen': -0.8024958372116089, 'rewards/rejected': -3.178739547729492, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.376243829727173, 'logps/chosen': -46.98686218261719, 'logps/rejected': -150.47857666015625, 'logps/ref_chosen': -33.495845794677734, 'logps/ref_rejected': -96.71635437011719, 'logits/chosen': -3.1882076263427734, 'logits/rejected': -3.236912488937378, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.05922134220600128, 'epsilon_dpo/loss_margin_mean': 40.271209716796875, 'epsilon_dpo/beta_margin_mean': 2.376243829727173, 'epsilon_dpo/beta_margin_std': 2.1569864749908447, 'epsilon_dpo/beta_margin_grad_mean': -0.18756245076656342, 'epsilon_dpo/beta_margin_grad_std': 0.23345990478992462, 'kl/beta': 0.059662893414497375, 'kl/avg_steps': 0.75, 'epoch': 0.12}
|
|||
|
|
12%|█████████████▊ | 82/681 [03:30<25:42, 2.57s/it]
12%|██████████████ | 83/681 [03:32<25:03, 2.51s/it]
{'loss': 0.424, 'grad_norm': 67.05672454833984, 'learning_rate': 4.994435419342304e-07, 'rewards/chosen': -0.6363058090209961, 'rewards/rejected': -3.1005678176879883, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.464262008666992, 'logps/chosen': -36.729583740234375, 'logps/rejected': -161.15139770507812, 'logps/ref_chosen': -25.916236877441406, 'logps/ref_rejected': -108.29981994628906, 'logits/chosen': -3.163329839706421, 'logits/rejected': -3.297056198120117, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.05872496962547302, 'epsilon_dpo/loss_margin_mean': 42.03822326660156, 'epsilon_dpo/beta_margin_mean': 2.464262008666992, 'epsilon_dpo/beta_margin_std': 1.7969151735305786, 'epsilon_dpo/beta_margin_grad_mean': -0.16398201882839203, 'epsilon_dpo/beta_margin_grad_std': 0.18266217410564423, 'kl/beta': 0.05921875312924385, 'kl/avg_steps': 0.84375, 'epoch': 0.12}
|
|||
|
|
12%|██████████████ | 83/681 [03:32<25:03, 2.51s/it]
12%|██████████████▏ | 84/681 [03:34<25:19, 2.55s/it]
{'loss': 0.4147, 'grad_norm': 68.96977996826172, 'learning_rate': 4.993546786148857e-07, 'rewards/chosen': -0.8531267642974854, 'rewards/rejected': -3.1109256744384766, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.2577991485595703, 'logps/chosen': -51.26725769042969, 'logps/rejected': -148.75570678710938, 'logps/ref_chosen': -36.62953567504883, 'logps/ref_rejected': -95.27814483642578, 'logits/chosen': -3.2171850204467773, 'logits/rejected': -3.2014272212982178, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.058215267956256866, 'epsilon_dpo/loss_margin_mean': 38.83984375, 'epsilon_dpo/beta_margin_mean': 2.257798910140991, 'epsilon_dpo/beta_margin_std': 1.5832382440567017, 'epsilon_dpo/beta_margin_grad_mean': -0.1662423461675644, 'epsilon_dpo/beta_margin_grad_std': 0.16180217266082764, 'kl/beta': 0.05872327461838722, 'kl/avg_steps': 0.875, 'epoch': 0.12}
|
|||
|
|
12%|██████████████▏ | 84/681 [03:34<25:19, 2.55s/it]
12%|██████████████▎ | 85/681 [03:37<25:23, 2.56s/it]
{'loss': 0.564, 'grad_norm': 81.10045623779297, 'learning_rate': 4.992592445678582e-07, 'rewards/chosen': -0.9453153610229492, 'rewards/rejected': -2.953948497772217, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.0086328983306885, 'logps/chosen': -59.8691291809082, 'logps/rejected': -135.07781982421875, 'logps/ref_chosen': -43.555397033691406, 'logps/ref_rejected': -83.9044418334961, 'logits/chosen': -3.2968966960906982, 'logits/rejected': -3.2394418716430664, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.05781027674674988, 'epsilon_dpo/loss_margin_mean': 34.859657287597656, 'epsilon_dpo/beta_margin_mean': 2.0086328983306885, 'epsilon_dpo/beta_margin_std': 1.7025460004806519, 'epsilon_dpo/beta_margin_grad_mean': -0.20659653842449188, 'epsilon_dpo/beta_margin_grad_std': 0.2083936482667923, 'kl/beta': 0.05821390450000763, 'kl/avg_steps': 0.703125, 'epoch': 0.12}
|
|||
|
|
12%|██████████████▎ | 85/681 [03:37<25:23, 2.56s/it]
13%|██████████████▌ | 86/681 [03:40<25:40, 2.59s/it]
{'loss': 0.6967, 'grad_norm': 107.6495132446289, 'learning_rate': 4.991572423079235e-07, 'rewards/chosen': -0.9202961921691895, 'rewards/rejected': -3.0485289096832275, 'rewards/accuracies': 0.875, 'rewards/margins': 2.128232479095459, 'logps/chosen': -54.787696838378906, 'logps/rejected': -145.7591094970703, 'logps/ref_chosen': -38.846839904785156, 'logps/ref_rejected': -92.59671020507812, 'logits/chosen': -3.272704601287842, 'logits/rejected': -3.2372689247131348, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.057433828711509705, 'epsilon_dpo/loss_margin_mean': 37.22154998779297, 'epsilon_dpo/beta_margin_mean': 2.128232479095459, 'epsilon_dpo/beta_margin_std': 2.417414903640747, 'epsilon_dpo/beta_margin_grad_mean': -0.2296195775270462, 'epsilon_dpo/beta_margin_grad_std': 0.22347640991210938, 'kl/beta': 0.05780744552612305, 'kl/avg_steps': 0.65625, 'epoch': 0.13}
|
|||
|
|
13%|██████████████▌ | 86/681 [03:40<25:40, 2.59s/it]
13%|██████████████▋ | 87/681 [03:42<25:18, 2.56s/it]
{'loss': 0.4717, 'grad_norm': 59.761165618896484, 'learning_rate': 4.990486745229364e-07, 'rewards/chosen': -0.8040882349014282, 'rewards/rejected': -3.161027193069458, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.3569388389587402, 'logps/chosen': -52.74066162109375, 'logps/rejected': -157.99281311035156, 'logps/ref_chosen': -38.653236389160156, 'logps/ref_rejected': -102.44976043701172, 'logits/chosen': -3.230192184448242, 'logits/rejected': -3.3376736640930176, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.05698757991194725, 'epsilon_dpo/loss_margin_mean': 41.45563507080078, 'epsilon_dpo/beta_margin_mean': 2.3569390773773193, 'epsilon_dpo/beta_margin_std': 1.9608224630355835, 'epsilon_dpo/beta_margin_grad_mean': -0.18504740297794342, 'epsilon_dpo/beta_margin_grad_std': 0.18343256413936615, 'kl/beta': 0.057430557906627655, 'kl/avg_steps': 0.78125, 'epoch': 0.13}
|
|||
|
|
13%|██████████████▋ | 87/681 [03:42<25:18, 2.56s/it]
13%|██████████████▊ | 88/681 [03:45<25:24, 2.57s/it]
{'loss': 0.4141, 'grad_norm': 58.7109375, 'learning_rate': 4.989335440737586e-07, 'rewards/chosen': -0.7879468202590942, 'rewards/rejected': -3.3477907180786133, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.5598437786102295, 'logps/chosen': -51.133506774902344, 'logps/rejected': -175.40699768066406, 'logps/ref_chosen': -37.23695373535156, 'logps/ref_rejected': -116.12947082519531, 'logits/chosen': -3.27421236038208, 'logits/rejected': -3.3343310356140137, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.05656362697482109, 'epsilon_dpo/loss_margin_mean': 45.38097381591797, 'epsilon_dpo/beta_margin_mean': 2.5598440170288086, 'epsilon_dpo/beta_margin_std': 1.910704493522644, 'epsilon_dpo/beta_margin_grad_mean': -0.1606673151254654, 'epsilon_dpo/beta_margin_grad_std': 0.18233220279216766, 'kl/beta': 0.05698535963892937, 'kl/avg_steps': 0.75, 'epoch': 0.13}
|
|||
|
|
13%|██████████████▊ | 88/681 [03:45<25:24, 2.57s/it]
13%|███████████████ | 89/681 [03:47<25:01, 2.54s/it]
{'loss': 0.4951, 'grad_norm': 69.53501892089844, 'learning_rate': 4.988118539941847e-07, 'rewards/chosen': -0.7197608947753906, 'rewards/rejected': -2.946448802947998, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.2266876697540283, 'logps/chosen': -52.141876220703125, 'logps/rejected': -140.5678253173828, 'logps/ref_chosen': -39.35747146606445, 'logps/ref_rejected': -88.01043701171875, 'logits/chosen': -3.2087016105651855, 'logits/rejected': -3.23443603515625, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.05614255368709564, 'epsilon_dpo/loss_margin_mean': 39.772979736328125, 'epsilon_dpo/beta_margin_mean': 2.2266876697540283, 'epsilon_dpo/beta_margin_std': 1.9912763833999634, 'epsilon_dpo/beta_margin_grad_mean': -0.18962043523788452, 'epsilon_dpo/beta_margin_grad_std': 0.18511323630809784, 'kl/beta': 0.056561149656772614, 'kl/avg_steps': 0.75, 'epoch': 0.13}
|
|||
|
|
13%|███████████████ | 89/681 [03:47<25:01, 2.54s/it]
13%|███████████████▏ | 90/681 [03:50<24:38, 2.50s/it]
{'loss': 0.5447, 'grad_norm': 91.28634643554688, 'learning_rate': 4.986836074908615e-07, 'rewards/chosen': -0.9231457710266113, 'rewards/rejected': -3.432866096496582, 'rewards/accuracies': 0.875, 'rewards/margins': 2.5097203254699707, 'logps/chosen': -46.801185607910156, 'logps/rejected': -181.02479553222656, 'logps/ref_chosen': -30.30811882019043, 'logps/ref_rejected': -119.35741424560547, 'logits/chosen': -3.129134178161621, 'logits/rejected': -3.3651089668273926, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.055777255445718765, 'epsilon_dpo/loss_margin_mean': 45.174312591552734, 'epsilon_dpo/beta_margin_mean': 2.5097200870513916, 'epsilon_dpo/beta_margin_std': 2.4309592247009277, 'epsilon_dpo/beta_margin_grad_mean': -0.19303679466247559, 'epsilon_dpo/beta_margin_grad_std': 0.21628044545650482, 'kl/beta': 0.056140098720788956, 'kl/avg_steps': 0.65625, 'epoch': 0.13}
|
|||
|
|
13%|███████████████▏ | 90/681 [03:50<24:38, 2.50s/it]
13%|███████████████▎ | 91/681 [03:52<24:26, 2.49s/it]
{'loss': 0.4213, 'grad_norm': 68.51940155029297, 'learning_rate': 4.985488079432037e-07, 'rewards/chosen': -0.8394383192062378, 'rewards/rejected': -3.1970739364624023, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.357635736465454, 'logps/chosen': -50.61489486694336, 'logps/rejected': -151.99273681640625, 'logps/ref_chosen': -35.484596252441406, 'logps/ref_rejected': -94.16378784179688, 'logits/chosen': -3.2310478687286377, 'logits/rejected': -3.2350800037384033, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.05532645061612129, 'epsilon_dpo/loss_margin_mean': 42.69865036010742, 'epsilon_dpo/beta_margin_mean': 2.357635736465454, 'epsilon_dpo/beta_margin_std': 1.7421871423721313, 'epsilon_dpo/beta_margin_grad_mean': -0.16726556420326233, 'epsilon_dpo/beta_margin_grad_std': 0.17135490477085114, 'kl/beta': 0.055774081498384476, 'kl/avg_steps': 0.8125, 'epoch': 0.13}
|
|||
|
|
13%|███████████████▎ | 91/681 [03:52<24:26, 2.49s/it]
14%|███████████████▌ | 92/681 [03:55<24:29, 2.49s/it]
{'loss': 0.5756, 'grad_norm': 87.93189239501953, 'learning_rate': 4.984074589033043e-07, 'rewards/chosen': -0.808807373046875, 'rewards/rejected': -3.0445470809936523, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.2357397079467773, 'logps/chosen': -52.659706115722656, 'logps/rejected': -139.8248291015625, 'logps/ref_chosen': -37.970062255859375, 'logps/ref_rejected': -84.28839111328125, 'logits/chosen': -3.121596574783325, 'logits/rejected': -3.1575112342834473, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.05489783734083176, 'epsilon_dpo/loss_margin_mean': 40.8467903137207, 'epsilon_dpo/beta_margin_mean': 2.2357397079467773, 'epsilon_dpo/beta_margin_std': 2.0685596466064453, 'epsilon_dpo/beta_margin_grad_mean': -0.2024046629667282, 'epsilon_dpo/beta_margin_grad_std': 0.220162495970726, 'kl/beta': 0.05532456934452057, 'kl/avg_steps': 0.78125, 'epoch': 0.14}
|
|||
|
|
14%|███████████████▌ | 92/681 [03:55<24:29, 2.49s/it]
14%|███████████████▋ | 93/681 [03:57<23:43, 2.42s/it]
{'loss': 0.4779, 'grad_norm': 66.98503875732422, 'learning_rate': 4.982595640958425e-07, 'rewards/chosen': -0.88683021068573, 'rewards/rejected': -3.1384482383728027, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.251617908477783, 'logps/chosen': -51.651126861572266, 'logps/rejected': -139.5394287109375, 'logps/ref_chosen': -35.3890266418457, 'logps/ref_rejected': -81.84159851074219, 'logits/chosen': -3.2347559928894043, 'logits/rejected': -3.2382359504699707, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.05447227507829666, 'epsilon_dpo/loss_margin_mean': 41.435726165771484, 'epsilon_dpo/beta_margin_mean': 2.251617908477783, 'epsilon_dpo/beta_margin_std': 1.9179073572158813, 'epsilon_dpo/beta_margin_grad_mean': -0.18667982518672943, 'epsilon_dpo/beta_margin_grad_std': 0.18083734810352325, 'kl/beta': 0.05489569902420044, 'kl/avg_steps': 0.78125, 'epoch': 0.14}
|
|||
|
|
14%|███████████████▋ | 93/681 [03:57<23:43, 2.42s/it]
14%|███████████████▊ | 94/681 [04:00<24:23, 2.49s/it]
{'loss': 0.3679, 'grad_norm': 63.9381103515625, 'learning_rate': 4.98105127417984e-07, 'rewards/chosen': -0.7756279706954956, 'rewards/rejected': -3.350586414337158, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.574958324432373, 'logps/chosen': -53.30908203125, 'logps/rejected': -168.24609375, 'logps/ref_chosen': -38.974853515625, 'logps/ref_rejected': -106.16789245605469, 'logits/chosen': -3.194207191467285, 'logits/rejected': -3.2492258548736572, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.05399893596768379, 'epsilon_dpo/loss_margin_mean': 47.74397659301758, 'epsilon_dpo/beta_margin_mean': 2.574958562850952, 'epsilon_dpo/beta_margin_std': 1.9459197521209717, 'epsilon_dpo/beta_margin_grad_mean': -0.15303176641464233, 'epsilon_dpo/beta_margin_grad_std': 0.14751987159252167, 'kl/beta': 0.05447014793753624, 'kl/avg_steps': 0.875, 'epoch': 0.14}
|
|||
|
|
14%|███████████████▊ | 94/681 [04:00<24:23, 2.49s/it]
14%|████████████████ | 95/681 [04:02<24:32, 2.51s/it]
{'loss': 0.4364, 'grad_norm': 63.63764572143555, 'learning_rate': 4.979441529392784e-07, 'rewards/chosen': -0.6584010124206543, 'rewards/rejected': -2.824158191680908, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.165757417678833, 'logps/chosen': -41.924564361572266, 'logps/rejected': -133.45326232910156, 'logps/ref_chosen': -29.644317626953125, 'logps/ref_rejected': -80.65695190429688, 'logits/chosen': -3.180877208709717, 'logits/rejected': -3.2767512798309326, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.05351366475224495, 'epsilon_dpo/loss_margin_mean': 40.51607131958008, 'epsilon_dpo/beta_margin_mean': 2.165757417678833, 'epsilon_dpo/beta_margin_std': 1.6699368953704834, 'epsilon_dpo/beta_margin_grad_mean': -0.176425963640213, 'epsilon_dpo/beta_margin_grad_std': 0.15948951244354248, 'kl/beta': 0.053997669368982315, 'kl/avg_steps': 0.90625, 'epoch': 0.14}
|
|||
|
|
14%|████████████████ | 95/681 [04:02<24:32, 2.51s/it]
14%|████████████████▏ | 96/681 [04:05<24:39, 2.53s/it]
{'loss': 0.4965, 'grad_norm': 85.53929901123047, 'learning_rate': 4.977766449015534e-07, 'rewards/chosen': -0.8906632661819458, 'rewards/rejected': -3.444911479949951, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.554248094558716, 'logps/chosen': -65.12588500976562, 'logps/rejected': -167.49996948242188, 'logps/ref_chosen': -48.42084884643555, 'logps/ref_rejected': -102.56741333007812, 'logits/chosen': -3.2188198566436768, 'logits/rejected': -3.264829635620117, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.05311667546629906, 'epsilon_dpo/loss_margin_mean': 48.22751998901367, 'epsilon_dpo/beta_margin_mean': 2.554248094558716, 'epsilon_dpo/beta_margin_std': 2.2736546993255615, 'epsilon_dpo/beta_margin_grad_mean': -0.18641680479049683, 'epsilon_dpo/beta_margin_grad_std': 0.20043620467185974, 'kl/beta': 0.05351271107792854, 'kl/avg_steps': 0.75, 'epoch': 0.14}
|
|||
|
|
14%|████████████████▏ | 96/681 [04:05<24:39, 2.53s/it]
14%|████████████████▍ | 97/681 [04:07<25:21, 2.61s/it]
{'loss': 0.3851, 'grad_norm': 52.84062194824219, 'learning_rate': 4.976026077188012e-07, 'rewards/chosen': -0.6797794103622437, 'rewards/rejected': -3.2887463569641113, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.6089670658111572, 'logps/chosen': -47.99535369873047, 'logps/rejected': -145.8551025390625, 'logps/ref_chosen': -35.1164436340332, 'logps/ref_rejected': -83.36341857910156, 'logits/chosen': -3.178095817565918, 'logits/rejected': -3.169985294342041, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.05267146974802017, 'epsilon_dpo/loss_margin_mean': 49.61277770996094, 'epsilon_dpo/beta_margin_mean': 2.608966827392578, 'epsilon_dpo/beta_margin_std': 1.9469996690750122, 'epsilon_dpo/beta_margin_grad_mean': -0.15445828437805176, 'epsilon_dpo/beta_margin_grad_std': 0.16520872712135315, 'kl/beta': 0.053114354610443115, 'kl/avg_steps': 0.84375, 'epoch': 0.14}
|
|||
|
|
14%|████████████████▍ | 97/681 [04:07<25:21, 2.61s/it]
14%|████████████████▌ | 98/681 [04:10<24:53, 2.56s/it]
{'loss': 0.5783, 'grad_norm': 101.56369018554688, 'learning_rate': 4.974220459770639e-07, 'rewards/chosen': -1.079300880432129, 'rewards/rejected': -3.3333208560943604, 'rewards/accuracies': 0.875, 'rewards/margins': 2.2540199756622314, 'logps/chosen': -65.423828125, 'logps/rejected': -165.55812072753906, 'logps/ref_chosen': -44.868499755859375, 'logps/ref_rejected': -101.7425537109375, 'logits/chosen': -3.108753204345703, 'logits/rejected': -3.2454357147216797, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.05228015407919884, 'epsilon_dpo/loss_margin_mean': 43.26023864746094, 'epsilon_dpo/beta_margin_mean': 2.2540199756622314, 'epsilon_dpo/beta_margin_std': 1.9207807779312134, 'epsilon_dpo/beta_margin_grad_mean': -0.19064751267433167, 'epsilon_dpo/beta_margin_grad_std': 0.22201648354530334, 'kl/beta': 0.0526699498295784, 'kl/avg_steps': 0.75, 'epoch': 0.14}
|
|||
|
|
14%|████████████████▌ | 98/681 [04:10<24:53, 2.56s/it]
15%|████████████████▋ | 99/681 [04:12<24:22, 2.51s/it]
{'loss': 0.3636, 'grad_norm': 56.95939254760742, 'learning_rate': 4.972349644343108e-07, 'rewards/chosen': -0.7500526905059814, 'rewards/rejected': -3.516953945159912, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.7669014930725098, 'logps/chosen': -40.343666076660156, 'logps/rejected': -158.97531127929688, 'logps/ref_chosen': -25.89197540283203, 'logps/ref_rejected': -91.06307983398438, 'logits/chosen': -3.0614562034606934, 'logits/rejected': -3.1564629077911377, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.051825616508722305, 'epsilon_dpo/loss_margin_mean': 53.46054458618164, 'epsilon_dpo/beta_margin_mean': 2.7669012546539307, 'epsilon_dpo/beta_margin_std': 2.086336135864258, 'epsilon_dpo/beta_margin_grad_mean': -0.1456242799758911, 'epsilon_dpo/beta_margin_grad_std': 0.16527007520198822, 'kl/beta': 0.05227786675095558, 'kl/avg_steps': 0.875, 'epoch': 0.15}
|
|||
|
|
15%|████████████████▋ | 99/681 [04:12<24:22, 2.51s/it]
15%|████████████████▋ | 100/681 [04:15<23:47, 2.46s/it]
{'loss': 0.6663, 'grad_norm': 109.34486389160156, 'learning_rate': 4.970413680203148e-07, 'rewards/chosen': -0.9696159362792969, 'rewards/rejected': -2.9393372535705566, 'rewards/accuracies': 0.828125, 'rewards/margins': 1.9697211980819702, 'logps/chosen': -60.350921630859375, 'logps/rejected': -136.6974334716797, 'logps/ref_chosen': -41.60627746582031, 'logps/ref_rejected': -79.52035522460938, 'logits/chosen': -3.0767757892608643, 'logits/rejected': -3.0463757514953613, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.05150564759969711, 'epsilon_dpo/loss_margin_mean': 38.432437896728516, 'epsilon_dpo/beta_margin_mean': 1.9697211980819702, 'epsilon_dpo/beta_margin_std': 1.9218071699142456, 'epsilon_dpo/beta_margin_grad_mean': -0.23118621110916138, 'epsilon_dpo/beta_margin_grad_std': 0.23917478322982788, 'kl/beta': 0.05182440206408501, 'kl/avg_steps': 0.625, 'epoch': 0.15}
|
|||
|
|
15%|████████████████▋ | 100/681 [04:15<23:47, 2.46s/it][INFO|trainer.py:4307] 2026-04-18 09:35:37,096 >>
|
|||
|
|
***** Running Evaluation *****
|
|||
|
|
[INFO|trainer.py:4309] 2026-04-18 09:35:37,096 >> Num examples = 2339
|
|||
|
|
[INFO|trainer.py:4312] 2026-04-18 09:35:37,096 >> Batch size = 8
|
|||
|
|
|
|||
|
|
0%| | 0/73 [00:00<?, ?it/s][A
|
|||
|
|
3%|███▏ | 2/73 [00:00<00:18, 3.86it/s][A
|
|||
|
|
4%|████▊ | 3/73 [00:01<00:27, 2.57it/s][A
|
|||
|
|
5%|██████▍ | 4/73 [00:01<00:30, 2.24it/s][A
|
|||
|
|
7%|████████ | 5/73 [00:02<00:32, 2.08it/s][A
|
|||
|
|
8%|█████████▌ | 6/73 [00:02<00:33, 2.01it/s][A
|
|||
|
|
10%|███████████▏ | 7/73 [00:03<00:33, 2.00it/s][A
|
|||
|
|
11%|████████████▊ | 8/73 [00:03<00:31, 2.06it/s][A
|
|||
|
|
12%|██████████████▍ | 9/73 [00:04<00:32, 2.00it/s][A
|
|||
|
|
14%|███████████████▉ | 10/73 [00:04<00:32, 1.95it/s][A
|
|||
|
|
15%|█████████████████▍ | 11/73 [00:05<00:31, 1.96it/s][A
|
|||
|
|
16%|███████████████████ | 12/73 [00:05<00:31, 1.96it/s][A
|
|||
|
|
18%|████████████████████▋ | 13/73 [00:06<00:31, 1.91it/s][A
|
|||
|
|
19%|██████████████████████▏ | 14/73 [00:06<00:30, 1.90it/s][A
|
|||
|
|
21%|███████████████████████▊ | 15/73 [00:07<00:30, 1.87it/s][A
|
|||
|
|
22%|█████████████████████████▍ | 16/73 [00:07<00:30, 1.85it/s][A
|
|||
|
|
23%|███████████████████████████ | 17/73 [00:08<00:29, 1.87it/s][A
|
|||
|
|
25%|████████████████████████████▌ | 18/73 [00:09<00:29, 1.84it/s][A
|
|||
|
|
26%|██████████████████████████████▏ | 19/73 [00:09<00:29, 1.84it/s][A
|
|||
|
|
27%|███████████████████████████████▊ | 20/73 [00:10<00:29, 1.82it/s][A
|
|||
|
|
29%|█████████████████████████████████▎ | 21/73 [00:10<00:28, 1.82it/s][A
|
|||
|
|
30%|██████████████████████████████████▉ | 22/73 [00:11<00:28, 1.81it/s][A
|
|||
|
|
32%|████████████████████████████████████▌ | 23/73 [00:11<00:27, 1.84it/s][A
|
|||
|
|
33%|██████████████████████████████████████▏ | 24/73 [00:12<00:26, 1.87it/s][A
|
|||
|
|
34%|███████████████████████████████████████▋ | 25/73 [00:12<00:25, 1.87it/s][A
|
|||
|
|
36%|█████████████████████████████████████████▎ | 26/73 [00:13<00:24, 1.90it/s][A
|
|||
|
|
37%|██████████████████████████████████████████▉ | 27/73 [00:13<00:22, 2.04it/s][A
|
|||
|
|
38%|████████████████████████████████████████████▍ | 28/73 [00:14<00:22, 2.03it/s][A
|
|||
|
|
40%|██████████████████████████████████████████████ | 29/73 [00:14<00:21, 2.00it/s][A
|
|||
|
|
41%|███████████████████████████████████████████████▋ | 30/73 [00:15<00:21, 1.96it/s][A
|
|||
|
|
42%|█████████████████████████████████████████████████▎ | 31/73 [00:15<00:22, 1.91it/s][A
|
|||
|
|
44%|██████████████████████████████████████████████████▊ | 32/73 [00:16<00:21, 1.93it/s][A
|
|||
|
|
45%|████████████████████████████████████████████████████▍ | 33/73 [00:16<00:20, 1.92it/s][A
|
|||
|
|
47%|██████████████████████████████████████████████████████ | 34/73 [00:17<00:19, 1.96it/s][A
|
|||
|
|
48%|███████████████████████████████████████████████████████▌ | 35/73 [00:17<00:20, 1.90it/s][A
|
|||
|
|
49%|█████████████████████████████████████████████████████████▏ | 36/73 [00:18<00:19, 1.91it/s][A
|
|||
|
|
51%|██████████████████████████████████████████████████████████▊ | 37/73 [00:18<00:19, 1.87it/s][A
|
|||
|
|
52%|████████████████████████████████████████████████████████████▍ | 38/73 [00:19<00:18, 1.93it/s][A
|
|||
|
|
53%|█████████████████████████████████████████████████████████████▉ | 39/73 [00:20<00:17, 1.92it/s][A
|
|||
|
|
55%|███████████████████████████████████████████████████████████████▌ | 40/73 [00:20<00:17, 1.92it/s][A
|
|||
|
|
56%|█████████████████████████████████████████████████████████████████▏ | 41/73 [00:21<00:16, 1.93it/s][A
|
|||
|
|
58%|██████████████████████████████████████████████████████████████████▋ | 42/73 [00:21<00:16, 1.90it/s][A
|
|||
|
|
59%|████████████████████████████████████████████████████████████████████▎ | 43/73 [00:22<00:16, 1.87it/s][A
|
|||
|
|
60%|█████████████████████████████████████████████████████████████████████▉ | 44/73 [00:22<00:15, 1.86it/s][A
|
|||
|
|
62%|███████████████████████████████████████████████████████████████████████▌ | 45/73 [00:23<00:15, 1.83it/s][A
|
|||
|
|
63%|█████████████████████████████████████████████████████████████████████████ | 46/73 [00:23<00:14, 1.87it/s][A
|
|||
|
|
64%|██████████████████████████████████████████████████████████████████████████▋ | 47/73 [00:24<00:13, 1.89it/s][A
|
|||
|
|
66%|████████████████████████████████████████████████████████████████████████████▎ | 48/73 [00:24<00:12, 1.93it/s][A
|
|||
|
|
67%|█████████████████████████████████████████████████████████████████████████████▊ | 49/73 [00:25<00:12, 1.88it/s][A
|
|||
|
|
68%|███████████████████████████████████████████████████████████████████████████████▍ | 50/73 [00:25<00:12, 1.86it/s][A
|
|||
|
|
70%|█████████████████████████████████████████████████████████████████████████████████ | 51/73 [00:26<00:11, 1.84it/s][A
|
|||
|
|
71%|██████████████████████████████████████████████████████████████████████████████████▋ | 52/73 [00:26<00:11, 1.82it/s][A
|
|||
|
|
73%|████████████████████████████████████████████████████████████████████████████████████▏ | 53/73 [00:27<00:10, 1.94it/s][A
|
|||
|
|
74%|█████████████████████████████████████████████████████████████████████████████████████▊ | 54/73 [00:27<00:09, 1.97it/s][A
|
|||
|
|
75%|███████████████████████████████████████████████████████████████████████████████████████▍ | 55/73 [00:28<00:08, 2.01it/s][A
|
|||
|
|
77%|████████████████████████████████████████████████████████████████████████████████████████▉ | 56/73 [00:28<00:08, 1.96it/s][A
|
|||
|
|
78%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 57/73 [00:29<00:08, 1.91it/s][A
|
|||
|
|
79%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 58/73 [00:29<00:07, 1.95it/s][A
|
|||
|
|
81%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 59/73 [00:30<00:07, 1.89it/s][A
|
|||
|
|
82%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 60/73 [00:31<00:06, 1.89it/s][A
|
|||
|
|
84%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 61/73 [00:31<00:06, 1.87it/s][A
|
|||
|
|
85%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 62/73 [00:32<00:05, 1.84it/s][A
|
|||
|
|
86%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 63/73 [00:32<00:05, 1.89it/s][A
|
|||
|
|
88%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 64/73 [00:33<00:04, 1.91it/s][A
|
|||
|
|
89%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 65/73 [00:33<00:04, 1.86it/s][A
|
|||
|
|
90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 66/73 [00:34<00:03, 1.86it/s][A
|
|||
|
|
92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 67/73 [00:34<00:03, 1.85it/s][A
|
|||
|
|
93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 68/73 [00:35<00:02, 1.86it/s][A
|
|||
|
|
95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 69/73 [00:35<00:02, 1.98it/s][A
|
|||
|
|
96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 70/73 [00:36<00:01, 1.92it/s][A
|
|||
|
|
97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 71/73 [00:36<00:01, 1.90it/s][A
|
|||
|
|
99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 72/73 [00:37<00:00, 1.88it/s][A
|
|||
|
|
100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 73/73 [00:37<00:00, 1.99it/s][A
|
|||
|
|
[A{'eval_loss': 0.49363189935684204, 'eval_runtime': 38.4605, 'eval_samples_per_second': 60.816, 'eval_steps_per_second': 1.924, 'eval_epsilon_dpo/beta': 0.051277898252010345, 'eval_epsilon_dpo/loss_margin_mean': 29.590499877929688, 'eval_epsilon_dpo/beta_margin_mean': 1.5045814514160156, 'eval_epsilon_dpo/beta_margin_std': 2.153695583343506, 'eval_epsilon_dpo/beta_margin_grad_mean': -0.30362221598625183, 'eval_epsilon_dpo/beta_margin_grad_std': 0.2585260272026062, 'eval_rewards/chosen': -1.4540938138961792, 'eval_rewards/rejected': -2.9586753845214844, 'eval_rewards/accuracies': 0.7555650472640991, 'eval_rewards/margins': 1.5045814514160156, 'eval_logps/chosen': -96.53782653808594, 'eval_logps/rejected': -149.91758728027344, 'eval_logps/ref_chosen': -68.29110717773438, 'eval_logps/ref_rejected': -92.08038330078125, 'eval_logits/chosen': -3.276595115661621, 'eval_logits/rejected': -3.2684147357940674, 'eval_kl/p_epsilon_steps': 0.7226027250289917, 'eval_kl/n_epsilon_steps': 0.2765410840511322, 'epoch': 0.15}
|
|||
|
|
15%|████████████████▋ | 100/681 [04:53<23:47, 2.46s/it]
|
|||
|
|
100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 73/73 [00:37<00:00, 1.99it/s][A
|
|||
|
|
[A
15%|████████████████▌ | 101/681 [04:56<2:15:21, 14.00s/it]
{'loss': 0.6759, 'grad_norm': 104.73916625976562, 'learning_rate': 4.968412618365215e-07, 'rewards/chosen': -1.0630580186843872, 'rewards/rejected': -3.0400502681732178, 'rewards/accuracies': 0.8125, 'rewards/margins': 1.976992130279541, 'logps/chosen': -67.021484375, 'logps/rejected': -148.63449096679688, 'logps/ref_chosen': -46.34172439575195, 'logps/ref_rejected': -89.14061737060547, 'logits/chosen': -3.2080070972442627, 'logits/rejected': -3.206873655319214, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.051185738295316696, 'epsilon_dpo/loss_margin_mean': 38.81412124633789, 'epsilon_dpo/beta_margin_mean': 1.9769922494888306, 'epsilon_dpo/beta_margin_std': 2.0824813842773438, 'epsilon_dpo/beta_margin_grad_mean': -0.23383933305740356, 'epsilon_dpo/beta_margin_grad_std': 0.23400039970874786, 'kl/beta': 0.05150251090526581, 'kl/avg_steps': 0.625, 'epoch': 0.15}
|
|||
|
|
15%|████████████████▌ | 101/681 [04:56<2:15:21, 14.00s/it]
15%|████████████████▊ | 102/681 [04:58<1:41:41, 10.54s/it]
{'loss': 0.5495, 'grad_norm': 70.99539184570312, 'learning_rate': 4.966346511559149e-07, 'rewards/chosen': -0.9300694465637207, 'rewards/rejected': -2.9457807540893555, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.0157113075256348, 'logps/chosen': -59.05921936035156, 'logps/rejected': -132.32525634765625, 'logps/ref_chosen': -40.793182373046875, 'logps/ref_rejected': -74.26336669921875, 'logits/chosen': -3.2234458923339844, 'logits/rejected': -3.1278076171875, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.05080382898449898, 'epsilon_dpo/loss_margin_mean': 39.79585647583008, 'epsilon_dpo/beta_margin_mean': 2.0157113075256348, 'epsilon_dpo/beta_margin_std': 1.7352498769760132, 'epsilon_dpo/beta_margin_grad_mean': -0.20627860724925995, 'epsilon_dpo/beta_margin_grad_std': 0.19808727502822876, 'kl/beta': 0.051182620227336884, 'kl/avg_steps': 0.75, 'epoch': 0.15}
|
|||
|
|
15%|████████████████▊ | 102/681 [04:58<1:41:41, 10.54s/it]
15%|████████████████▉ | 103/681 [05:01<1:18:30, 8.15s/it]
{'loss': 0.3972, 'grad_norm': 70.43810272216797, 'learning_rate': 4.964215414228785e-07, 'rewards/chosen': -0.8012863397598267, 'rewards/rejected': -3.6245927810668945, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.823306083679199, 'logps/chosen': -53.713714599609375, 'logps/rejected': -173.4473876953125, 'logps/ref_chosen': -37.85403823852539, 'logps/ref_rejected': -101.43891906738281, 'logits/chosen': -3.152428388595581, 'logits/rejected': -3.057560920715332, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.05037800967693329, 'epsilon_dpo/loss_margin_mean': 56.148807525634766, 'epsilon_dpo/beta_margin_mean': 2.8233063220977783, 'epsilon_dpo/beta_margin_std': 2.1322638988494873, 'epsilon_dpo/beta_margin_grad_mean': -0.14925150573253632, 'epsilon_dpo/beta_margin_grad_std': 0.1886918544769287, 'kl/beta': 0.050801608711481094, 'kl/avg_steps': 0.84375, 'epoch': 0.15}
|
|||
|
|
15%|████████████████▉ | 103/681 [05:01<1:18:30, 8.15s/it]
15%|█████████████████ | 104/681 [05:03<1:01:29, 6.39s/it]
{'loss': 0.4096, 'grad_norm': 61.21420669555664, 'learning_rate': 4.96201938253052e-07, 'rewards/chosen': -0.769996166229248, 'rewards/rejected': -3.549788475036621, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.779792308807373, 'logps/chosen': -51.264808654785156, 'logps/rejected': -168.14761352539062, 'logps/ref_chosen': -35.89470672607422, 'logps/ref_rejected': -97.03848266601562, 'logits/chosen': -3.150313377380371, 'logits/rejected': -3.136101245880127, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.04997224360704422, 'epsilon_dpo/loss_margin_mean': 55.73902893066406, 'epsilon_dpo/beta_margin_mean': 2.779792547225952, 'epsilon_dpo/beta_margin_std': 2.224303960800171, 'epsilon_dpo/beta_margin_grad_mean': -0.16119545698165894, 'epsilon_dpo/beta_margin_grad_std': 0.1801510900259018, 'kl/beta': 0.05037655681371689, 'kl/avg_steps': 0.8125, 'epoch': 0.15}
|
|||
|
|
15%|█████████████████ | 104/681 [05:03<1:01:29, 6.39s/it]
15%|█████████████████▌ | 105/681 [05:05<49:36, 5.17s/it]
{'loss': 0.3212, 'grad_norm': 61.274295806884766, 'learning_rate': 4.959758474331832e-07, 'rewards/chosen': -0.8625220060348511, 'rewards/rejected': -3.8436279296875, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.9811058044433594, 'logps/chosen': -56.80089569091797, 'logps/rejected': -182.7860565185547, 'logps/ref_chosen': -39.414154052734375, 'logps/ref_rejected': -105.14703369140625, 'logits/chosen': -3.1032373905181885, 'logits/rejected': -3.1879420280456543, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.04952263832092285, 'epsilon_dpo/loss_margin_mean': 60.252281188964844, 'epsilon_dpo/beta_margin_mean': 2.9811060428619385, 'epsilon_dpo/beta_margin_std': 2.177426338195801, 'epsilon_dpo/beta_margin_grad_mean': -0.12934987246990204, 'epsilon_dpo/beta_margin_grad_std': 0.15336252748966217, 'kl/beta': 0.04997054487466812, 'kl/avg_steps': 0.90625, 'epoch': 0.15}
|
|||
|
|
15%|█████████████████▌ | 105/681 [05:05<49:36, 5.17s/it]
16%|█████████████████▋ | 106/681 [05:08<42:13, 4.41s/it]
{'loss': 0.4771, 'grad_norm': 60.775062561035156, 'learning_rate': 4.957432749209755e-07, 'rewards/chosen': -0.9345280528068542, 'rewards/rejected': -3.2354397773742676, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.3009119033813477, 'logps/chosen': -58.34198760986328, 'logps/rejected': -155.92471313476562, 'logps/ref_chosen': -39.36569595336914, 'logps/ref_rejected': -90.00059509277344, 'logits/chosen': -3.1399245262145996, 'logits/rejected': -3.1790456771850586, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.04914744198322296, 'epsilon_dpo/loss_margin_mean': 46.947818756103516, 'epsilon_dpo/beta_margin_mean': 2.3009119033813477, 'epsilon_dpo/beta_margin_std': 1.8214956521987915, 'epsilon_dpo/beta_margin_grad_mean': -0.18032847344875336, 'epsilon_dpo/beta_margin_grad_std': 0.19534119963645935, 'kl/beta': 0.049521755427122116, 'kl/avg_steps': 0.765625, 'epoch': 0.16}
|
|||
|
|
16%|█████████████████▋ | 106/681 [05:08<42:13, 4.41s/it]
16%|█████████████████▉ | 107/681 [05:10<36:19, 3.80s/it]
{'loss': 0.383, 'grad_norm': 64.9974365234375, 'learning_rate': 4.955042268449307e-07, 'rewards/chosen': -0.7801457643508911, 'rewards/rejected': -3.6794204711914062, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.8992745876312256, 'logps/chosen': -56.38127899169922, 'logps/rejected': -176.49288940429688, 'logps/ref_chosen': -40.38301086425781, 'logps/ref_rejected': -100.88917541503906, 'logits/chosen': -3.1247940063476562, 'logits/rejected': -3.1252992153167725, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.04868961498141289, 'epsilon_dpo/loss_margin_mean': 59.605445861816406, 'epsilon_dpo/beta_margin_mean': 2.8992745876312256, 'epsilon_dpo/beta_margin_std': 2.3069019317626953, 'epsilon_dpo/beta_margin_grad_mean': -0.14175310730934143, 'epsilon_dpo/beta_margin_grad_std': 0.17217276990413666, 'kl/beta': 0.04914548620581627, 'kl/avg_steps': 0.9375, 'epoch': 0.16}
|
|||
|
|
16%|█████████████████▉ | 107/681 [05:10<36:19, 3.80s/it]
16%|██████████████████ | 108/681 [05:13<32:39, 3.42s/it]
{'loss': 0.3473, 'grad_norm': 53.83271026611328, 'learning_rate': 4.952587095041881e-07, 'rewards/chosen': -0.8619762063026428, 'rewards/rejected': -3.8742575645446777, 'rewards/accuracies': 0.953125, 'rewards/margins': 3.0122811794281006, 'logps/chosen': -49.35261917114258, 'logps/rejected': -184.76495361328125, 'logps/ref_chosen': -31.513410568237305, 'logps/ref_rejected': -104.42971801757812, 'logits/chosen': -3.1003808975219727, 'logits/rejected': -3.143059492111206, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.048267822712659836, 'epsilon_dpo/loss_margin_mean': 62.49602508544922, 'epsilon_dpo/beta_margin_mean': 3.0122811794281006, 'epsilon_dpo/beta_margin_std': 2.167310953140259, 'epsilon_dpo/beta_margin_grad_mean': -0.13621723651885986, 'epsilon_dpo/beta_margin_grad_std': 0.16647490859031677, 'kl/beta': 0.04868902638554573, 'kl/avg_steps': 0.875, 'epoch': 0.16}
|
|||
|
|
16%|██████████████████ | 108/681 [05:13<32:39, 3.42s/it]
16%|██████████████████▏ | 109/681 [05:15<30:14, 3.17s/it]
{'loss': 0.5342, 'grad_norm': 100.95081329345703, 'learning_rate': 4.95006729368358e-07, 'rewards/chosen': -0.8883336782455444, 'rewards/rejected': -3.539311408996582, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.650977611541748, 'logps/chosen': -64.14362335205078, 'logps/rejected': -178.133544921875, 'logps/ref_chosen': -45.64976501464844, 'logps/ref_rejected': -104.12217712402344, 'logits/chosen': -3.113433361053467, 'logits/rejected': -3.186359167098999, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.04787931218743324, 'epsilon_dpo/loss_margin_mean': 55.51750564575195, 'epsilon_dpo/beta_margin_mean': 2.650977611541748, 'epsilon_dpo/beta_margin_std': 2.2566730976104736, 'epsilon_dpo/beta_margin_grad_mean': -0.17085351049900055, 'epsilon_dpo/beta_margin_grad_std': 0.21672260761260986, 'kl/beta': 0.0482666902244091, 'kl/avg_steps': 0.8125, 'epoch': 0.16}
|
|||
|
|
16%|██████████████████▏ | 109/681 [05:15<30:14, 3.17s/it]
16%|██████████████████▍ | 110/681 [05:18<28:17, 2.97s/it]
{'loss': 0.5765, 'grad_norm': 87.10065460205078, 'learning_rate': 4.947482930773511e-07, 'rewards/chosen': -1.1659729480743408, 'rewards/rejected': -3.349276065826416, 'rewards/accuracies': 0.84375, 'rewards/margins': 2.1833033561706543, 'logps/chosen': -74.82107543945312, 'logps/rejected': -155.9808349609375, 'logps/ref_chosen': -50.388492584228516, 'logps/ref_rejected': -85.45807647705078, 'logits/chosen': -3.0851223468780518, 'logits/rejected': -3.0515449047088623, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.04758320748806, 'epsilon_dpo/loss_margin_mean': 46.09016799926758, 'epsilon_dpo/beta_margin_mean': 2.1833033561706543, 'epsilon_dpo/beta_margin_std': 2.0075676441192627, 'epsilon_dpo/beta_margin_grad_mean': -0.20529796183109283, 'epsilon_dpo/beta_margin_grad_std': 0.21916480362415314, 'kl/beta': 0.047877684235572815, 'kl/avg_steps': 0.625, 'epoch': 0.16}
|
|||
|
|
16%|██████████████████▍ | 110/681 [05:18<28:17, 2.97s/it]
16%|██████████████████▌ | 111/681 [05:20<27:02, 2.85s/it]
{'loss': 0.4814, 'grad_norm': 77.24600219726562, 'learning_rate': 4.944834074412042e-07, 'rewards/chosen': -1.1880950927734375, 'rewards/rejected': -3.8150076866149902, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.6269125938415527, 'logps/chosen': -75.80860900878906, 'logps/rejected': -184.10238647460938, 'logps/ref_chosen': -50.71482849121094, 'logps/ref_rejected': -103.22978210449219, 'logits/chosen': -3.054626941680908, 'logits/rejected': -3.125295877456665, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.04724304750561714, 'epsilon_dpo/loss_margin_mean': 55.77882766723633, 'epsilon_dpo/beta_margin_mean': 2.6269123554229736, 'epsilon_dpo/beta_margin_std': 2.2809414863586426, 'epsilon_dpo/beta_margin_grad_mean': -0.17633913457393646, 'epsilon_dpo/beta_margin_grad_std': 0.20233316719532013, 'kl/beta': 0.047580309212207794, 'kl/avg_steps': 0.71875, 'epoch': 0.16}
|
|||
|
|
16%|██████████████████▌ | 111/681 [05:20<27:02, 2.85s/it]
16%|██████████████████▋ | 112/681 [05:23<25:57, 2.74s/it]
{'loss': 0.5174, 'grad_norm': 59.31022262573242, 'learning_rate': 4.942120794399002e-07, 'rewards/chosen': -0.8925973176956177, 'rewards/rejected': -2.976914405822754, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.084317207336426, 'logps/chosen': -50.82137680053711, 'logps/rejected': -132.23062133789062, 'logps/ref_chosen': -31.809959411621094, 'logps/ref_rejected': -68.63668823242188, 'logits/chosen': -3.0048656463623047, 'logits/rejected': -3.0176172256469727, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.04686162248253822, 'epsilon_dpo/loss_margin_mean': 44.58252716064453, 'epsilon_dpo/beta_margin_mean': 2.084317207336426, 'epsilon_dpo/beta_margin_std': 1.6938108205795288, 'epsilon_dpo/beta_margin_grad_mean': -0.19428886473178864, 'epsilon_dpo/beta_margin_grad_std': 0.19283361732959747, 'kl/beta': 0.04724076762795448, 'kl/avg_steps': 0.8125, 'epoch': 0.16}
|
|||
|
|
16%|██████████████████▋ | 112/681 [05:23<25:57, 2.74s/it]
17%|██████████████████▉ | 113/681 [05:25<25:17, 2.67s/it]
{'loss': 0.4213, 'grad_norm': 57.292938232421875, 'learning_rate': 4.939343162231841e-07, 'rewards/chosen': -1.0600247383117676, 'rewards/rejected': -3.3343217372894287, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.274296760559082, 'logps/chosen': -64.85562896728516, 'logps/rejected': -155.20111083984375, 'logps/ref_chosen': -42.08271408081055, 'logps/ref_rejected': -83.40155029296875, 'logits/chosen': -3.054755687713623, 'logits/rejected': -3.0067625045776367, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.04648394137620926, 'epsilon_dpo/loss_margin_mean': 49.026641845703125, 'epsilon_dpo/beta_margin_mean': 2.274296760559082, 'epsilon_dpo/beta_margin_std': 1.6202584505081177, 'epsilon_dpo/beta_margin_grad_mean': -0.1680552214384079, 'epsilon_dpo/beta_margin_grad_std': 0.16932794451713562, 'kl/beta': 0.04686002805829048, 'kl/avg_steps': 0.8125, 'epoch': 0.17}
|
|||
|
|
17%|██████████████████▉ | 113/681 [05:25<25:17, 2.67s/it]
17%|███████████████████ | 114/681 [05:28<23:59, 2.54s/it]
{'loss': 0.4844, 'grad_norm': 70.24754333496094, 'learning_rate': 4.936501251103751e-07, 'rewards/chosen': -0.9574916362762451, 'rewards/rejected': -3.5459303855895996, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.5884387493133545, 'logps/chosen': -63.09971237182617, 'logps/rejected': -168.25796508789062, 'logps/ref_chosen': -42.380760192871094, 'logps/ref_rejected': -91.27511596679688, 'logits/chosen': -2.998676300048828, 'logits/rejected': -3.029796600341797, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.04610930010676384, 'epsilon_dpo/loss_margin_mean': 56.26390075683594, 'epsilon_dpo/beta_margin_mean': 2.5884387493133545, 'epsilon_dpo/beta_margin_std': 2.3022119998931885, 'epsilon_dpo/beta_margin_grad_mean': -0.1812613159418106, 'epsilon_dpo/beta_margin_grad_std': 0.20109733939170837, 'kl/beta': 0.04648235812783241, 'kl/avg_steps': 0.8125, 'epoch': 0.17}
|
|||
|
|
17%|███████████████████ | 114/681 [05:28<23:59, 2.54s/it]
17%|███████████████████▎ | 115/681 [05:30<24:00, 2.55s/it]
{'loss': 0.4575, 'grad_norm': 66.69357299804688, 'learning_rate': 4.933595135901732e-07, 'rewards/chosen': -0.9589067697525024, 'rewards/rejected': -3.5014431476593018, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.5425362586975098, 'logps/chosen': -61.14916229248047, 'logps/rejected': -182.99166870117188, 'logps/ref_chosen': -40.20535659790039, 'logps/ref_rejected': -106.33647155761719, 'logits/chosen': -3.068523406982422, 'logits/rejected': -3.093740463256836, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.04570886492729187, 'epsilon_dpo/loss_margin_mean': 55.71139144897461, 'epsilon_dpo/beta_margin_mean': 2.5425362586975098, 'epsilon_dpo/beta_margin_std': 1.9873340129852295, 'epsilon_dpo/beta_margin_grad_mean': -0.16286557912826538, 'epsilon_dpo/beta_margin_grad_std': 0.18301641941070557, 'kl/beta': 0.046107735484838486, 'kl/avg_steps': 0.875, 'epoch': 0.17}
|
|||
|
|
17%|███████████████████▎ | 115/681 [05:30<24:00, 2.55s/it]
17%|███████████████████▍ | 116/681 [05:33<23:17, 2.47s/it]
{'loss': 0.4288, 'grad_norm': 69.93063354492188, 'learning_rate': 4.930624893204624e-07, 'rewards/chosen': -0.9880549907684326, 'rewards/rejected': -3.3488869667053223, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.3608322143554688, 'logps/chosen': -52.52592086791992, 'logps/rejected': -158.97201538085938, 'logps/ref_chosen': -30.788612365722656, 'logps/ref_rejected': -85.05826568603516, 'logits/chosen': -2.8386454582214355, 'logits/rejected': -2.940783977508545, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.04532666504383087, 'epsilon_dpo/loss_margin_mean': 52.17644500732422, 'epsilon_dpo/beta_margin_mean': 2.3608319759368896, 'epsilon_dpo/beta_margin_std': 1.805658221244812, 'epsilon_dpo/beta_margin_grad_mean': -0.17079366743564606, 'epsilon_dpo/beta_margin_grad_std': 0.1683167964220047, 'kl/beta': 0.04570779204368591, 'kl/avg_steps': 0.84375, 'epoch': 0.17}
|
|||
|
|
17%|███████████████████▍ | 116/681 [05:33<23:17, 2.47s/it]
17%|███████████████████▌ | 117/681 [05:35<23:08, 2.46s/it]
{'loss': 0.4764, 'grad_norm': 82.57999420166016, 'learning_rate': 4.927590601281083e-07, 'rewards/chosen': -1.2893564701080322, 'rewards/rejected': -3.461637020111084, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.1722805500030518, 'logps/chosen': -77.98645782470703, 'logps/rejected': -149.6578369140625, 'logps/ref_chosen': -49.41318893432617, 'logps/ref_rejected': -72.67740631103516, 'logits/chosen': -2.9002761840820312, 'logits/rejected': -2.8406801223754883, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.04501824453473091, 'epsilon_dpo/loss_margin_mean': 48.40717697143555, 'epsilon_dpo/beta_margin_mean': 2.1722805500030518, 'epsilon_dpo/beta_margin_std': 1.6980078220367432, 'epsilon_dpo/beta_margin_grad_mean': -0.18530869483947754, 'epsilon_dpo/beta_margin_grad_std': 0.18177969753742218, 'kl/beta': 0.04532535746693611, 'kl/avg_steps': 0.6875, 'epoch': 0.17}
|
|||
|
|
17%|███████████████████▌ | 117/681 [05:35<23:08, 2.46s/it]
17%|███████████████████▊ | 118/681 [05:37<23:20, 2.49s/it]
{'loss': 0.3965, 'grad_norm': 55.6863899230957, 'learning_rate': 4.924492340087524e-07, 'rewards/chosen': -1.0198127031326294, 'rewards/rejected': -3.47811222076416, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.458299398422241, 'logps/chosen': -58.43132400512695, 'logps/rejected': -158.32205200195312, 'logps/ref_chosen': -35.58433532714844, 'logps/ref_rejected': -80.31232452392578, 'logits/chosen': -2.8982632160186768, 'logits/rejected': -2.9912490844726562, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.044626448303461075, 'epsilon_dpo/loss_margin_mean': 55.162723541259766, 'epsilon_dpo/beta_margin_mean': 2.458299398422241, 'epsilon_dpo/beta_margin_std': 1.716274380683899, 'epsilon_dpo/beta_margin_grad_mean': -0.15689337253570557, 'epsilon_dpo/beta_margin_grad_std': 0.17096935212612152, 'kl/beta': 0.04501587525010109, 'kl/avg_steps': 0.875, 'epoch': 0.17}
|
|||
|
|
17%|███████████████████▊ | 118/681 [05:38<23:20, 2.49s/it]
17%|███████████████████▉ | 119/681 [05:40<23:39, 2.53s/it]
{'loss': 0.5632, 'grad_norm': 105.04483795166016, 'learning_rate': 4.92133019126601e-07, 'rewards/chosen': -1.2193212509155273, 'rewards/rejected': -3.855588912963867, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.63626766204834, 'logps/chosen': -68.64191436767578, 'logps/rejected': -196.2698974609375, 'logps/ref_chosen': -41.200706481933594, 'logps/ref_rejected': -109.11971282958984, 'logits/chosen': -2.9290332794189453, 'logits/rejected': -2.974984645843506, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.04427415132522583, 'epsilon_dpo/loss_margin_mean': 59.708961486816406, 'epsilon_dpo/beta_margin_mean': 2.63626766204834, 'epsilon_dpo/beta_margin_std': 2.4180660247802734, 'epsilon_dpo/beta_margin_grad_mean': -0.16944344341754913, 'epsilon_dpo/beta_margin_grad_std': 0.20828606188297272, 'kl/beta': 0.04462540149688721, 'kl/avg_steps': 0.796875, 'epoch': 0.17}
|
|||
|
|
17%|███████████████████▉ | 119/681 [05:40<23:39, 2.53s/it]
18%|████████████████████ | 120/681 [05:43<23:59, 2.57s/it]
{'loss': 0.3704, 'grad_norm': 58.40270233154297, 'learning_rate': 4.918104238142103e-07, 'rewards/chosen': -1.1369682550430298, 'rewards/rejected': -4.075782775878906, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.938814163208008, 'logps/chosen': -79.7890625, 'logps/rejected': -207.2266082763672, 'logps/ref_chosen': -53.9609489440918, 'logps/ref_rejected': -114.34416961669922, 'logits/chosen': -2.980833053588867, 'logits/rejected': -2.9574685096740723, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.043917279690504074, 'epsilon_dpo/loss_margin_mean': 67.05432891845703, 'epsilon_dpo/beta_margin_mean': 2.938814163208008, 'epsilon_dpo/beta_margin_std': 2.2117486000061035, 'epsilon_dpo/beta_margin_grad_mean': -0.1446925848722458, 'epsilon_dpo/beta_margin_grad_std': 0.17125862836837769, 'kl/beta': 0.04427260532975197, 'kl/avg_steps': 0.8125, 'epoch': 0.18}
|
|||
|
|
18%|████████████████████ | 120/681 [05:43<23:59, 2.57s/it]
18%|████████████████████▎ | 121/681 [05:45<24:11, 2.59s/it]
{'loss': 0.3024, 'grad_norm': 43.766937255859375, 'learning_rate': 4.91481456572267e-07, 'rewards/chosen': -1.0692138671875, 'rewards/rejected': -4.200495719909668, 'rewards/accuracies': 0.984375, 'rewards/margins': 3.131281852722168, 'logps/chosen': -57.087093353271484, 'logps/rejected': -215.80718994140625, 'logps/ref_chosen': -32.518829345703125, 'logps/ref_rejected': -119.22189331054688, 'logits/chosen': -2.7587661743164062, 'logits/rejected': -2.9042224884033203, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.04350843280553818, 'epsilon_dpo/loss_margin_mean': 72.01702117919922, 'epsilon_dpo/beta_margin_mean': 3.131282091140747, 'epsilon_dpo/beta_margin_std': 2.3148951530456543, 'epsilon_dpo/beta_margin_grad_mean': -0.12654456496238708, 'epsilon_dpo/beta_margin_grad_std': 0.1407029926776886, 'kl/beta': 0.04391578957438469, 'kl/avg_steps': 0.9375, 'epoch': 0.18}
|
|||
|
|
18%|████████████████████▎ | 121/681 [05:45<24:11, 2.59s/it]
18%|████████████████████▍ | 122/681 [05:48<23:46, 2.55s/it]
{'loss': 0.3434, 'grad_norm': 72.01390075683594, 'learning_rate': 4.911461260693638e-07, 'rewards/chosen': -1.1941652297973633, 'rewards/rejected': -4.377148151397705, 'rewards/accuracies': 0.921875, 'rewards/margins': 3.182982921600342, 'logps/chosen': -59.01881790161133, 'logps/rejected': -215.54666137695312, 'logps/ref_chosen': -31.410724639892578, 'logps/ref_rejected': -114.03694152832031, 'logits/chosen': -2.679837942123413, 'logits/rejected': -2.8655993938446045, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.04314511641860008, 'epsilon_dpo/loss_margin_mean': 73.9016342163086, 'epsilon_dpo/beta_margin_mean': 3.182982921600342, 'epsilon_dpo/beta_margin_std': 2.4074745178222656, 'epsilon_dpo/beta_margin_grad_mean': -0.13571257889270782, 'epsilon_dpo/beta_margin_grad_std': 0.1690312772989273, 'kl/beta': 0.0435079000890255, 'kl/avg_steps': 0.84375, 'epoch': 0.18}
|
|||
|
|
18%|████████████████████▍ | 122/681 [05:48<23:46, 2.55s/it]
18%|████████████████████▌ | 123/681 [05:50<23:24, 2.52s/it]
{'loss': 0.4491, 'grad_norm': 75.45635223388672, 'learning_rate': 4.908044411417711e-07, 'rewards/chosen': -1.1574070453643799, 'rewards/rejected': -3.8119406700134277, 'rewards/accuracies': 0.875, 'rewards/margins': 2.654533624649048, 'logps/chosen': -72.34667205810547, 'logps/rejected': -184.47792053222656, 'logps/ref_chosen': -45.39820098876953, 'logps/ref_rejected': -95.36239624023438, 'logits/chosen': -2.7718982696533203, 'logits/rejected': -2.8315348625183105, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.04283806309103966, 'epsilon_dpo/loss_margin_mean': 62.16706085205078, 'epsilon_dpo/beta_margin_mean': 2.654533624649048, 'epsilon_dpo/beta_margin_std': 2.180321216583252, 'epsilon_dpo/beta_margin_grad_mean': -0.16673751175403595, 'epsilon_dpo/beta_margin_grad_std': 0.19917066395282745, 'kl/beta': 0.043143875896930695, 'kl/avg_steps': 0.71875, 'epoch': 0.18}
|
|||
|
|
18%|████████████████████▌ | 123/681 [05:50<23:24, 2.52s/it]
18%|████████████████████▊ | 124/681 [05:53<23:29, 2.53s/it]
{'loss': 0.2979, 'grad_norm': 42.3536376953125, 'learning_rate': 4.904564107932048e-07, 'rewards/chosen': -1.1055363416671753, 'rewards/rejected': -4.310197830200195, 'rewards/accuracies': 0.96875, 'rewards/margins': 3.2046613693237305, 'logps/chosen': -68.72981262207031, 'logps/rejected': -226.0880126953125, 'logps/ref_chosen': -42.682899475097656, 'logps/ref_rejected': -124.48335266113281, 'logits/chosen': -2.7853384017944336, 'logits/rejected': -2.904179096221924, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.04243864864110947, 'epsilon_dpo/loss_margin_mean': 75.55775451660156, 'epsilon_dpo/beta_margin_mean': 3.2046613693237305, 'epsilon_dpo/beta_margin_std': 2.2337818145751953, 'epsilon_dpo/beta_margin_grad_mean': -0.12405575811862946, 'epsilon_dpo/beta_margin_grad_std': 0.14530256390571594, 'kl/beta': 0.04283599182963371, 'kl/avg_steps': 0.9375, 'epoch': 0.18}
|
|||
|
|
18%|████████████████████▊ | 124/681 [05:53<23:29, 2.53s/it]
18%|████████████████████▉ | 125/681 [05:55<23:34, 2.54s/it]
{'loss': 0.4111, 'grad_norm': 81.41658020019531, 'learning_rate': 4.90102044194588e-07, 'rewards/chosen': -1.3759876489639282, 'rewards/rejected': -4.034554958343506, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.658567428588867, 'logps/chosen': -63.22058868408203, 'logps/rejected': -184.8516845703125, 'logps/ref_chosen': -30.618457794189453, 'logps/ref_rejected': -88.96886444091797, 'logits/chosen': -2.5554890632629395, 'logits/rejected': -2.767099618911743, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.04211079701781273, 'epsilon_dpo/loss_margin_mean': 63.280696868896484, 'epsilon_dpo/beta_margin_mean': 2.658567428588867, 'epsilon_dpo/beta_margin_std': 2.0561933517456055, 'epsilon_dpo/beta_margin_grad_mean': -0.16087745130062103, 'epsilon_dpo/beta_margin_grad_std': 0.1763782650232315, 'kl/beta': 0.04243813455104828, 'kl/avg_steps': 0.78125, 'epoch': 0.18}
|
|||
|
|
18%|████████████████████▉ | 125/681 [05:55<23:34, 2.54s/it]
19%|█████████████████████ | 126/681 [05:58<23:37, 2.55s/it]
{'loss': 0.4561, 'grad_norm': 81.50450897216797, 'learning_rate': 4.897413506838102e-07, 'rewards/chosen': -1.4002439975738525, 'rewards/rejected': -4.327142715454102, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.926898956298828, 'logps/chosen': -72.39849090576172, 'logps/rejected': -208.291748046875, 'logps/ref_chosen': -38.92976379394531, 'logps/ref_rejected': -104.5777587890625, 'logits/chosen': -2.6770665645599365, 'logits/rejected': -2.785977840423584, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.041758038103580475, 'epsilon_dpo/loss_margin_mean': 70.24527740478516, 'epsilon_dpo/beta_margin_mean': 2.926898956298828, 'epsilon_dpo/beta_margin_std': 2.228339433670044, 'epsilon_dpo/beta_margin_grad_mean': -0.1536438912153244, 'epsilon_dpo/beta_margin_grad_std': 0.21458838880062103, 'kl/beta': 0.042109157890081406, 'kl/avg_steps': 0.84375, 'epoch': 0.19}
|
|||
|
|
19%|█████████████████████ | 126/681 [05:58<23:37, 2.55s/it]
19%|█████████████████████▎ | 127/681 [06:01<23:43, 2.57s/it]
{'loss': 0.395, 'grad_norm': 61.22636413574219, 'learning_rate': 4.89374339765481e-07, 'rewards/chosen': -1.2484067678451538, 'rewards/rejected': -3.5723307132720947, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.3239240646362305, 'logps/chosen': -69.19281005859375, 'logps/rejected': -167.935791015625, 'logps/ref_chosen': -39.06876754760742, 'logps/ref_rejected': -81.60107421875, 'logits/chosen': -2.705301523208618, 'logits/rejected': -2.7008934020996094, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.04140865430235863, 'epsilon_dpo/loss_margin_mean': 56.2106819152832, 'epsilon_dpo/beta_margin_mean': 2.3239240646362305, 'epsilon_dpo/beta_margin_std': 1.5966986417770386, 'epsilon_dpo/beta_margin_grad_mean': -0.1614929735660553, 'epsilon_dpo/beta_margin_grad_std': 0.1582755744457245, 'kl/beta': 0.04175683483481407, 'kl/avg_steps': 0.84375, 'epoch': 0.19}
|
|||
|
|
19%|█████████████████████▎ | 127/681 [06:01<23:43, 2.57s/it]
19%|█████████████████████▍ | 128/681 [06:03<23:42, 2.57s/it]
{'loss': 0.5065, 'grad_norm': 69.2333984375, 'learning_rate': 4.890010211106795e-07, 'rewards/chosen': -1.3495298624038696, 'rewards/rejected': -3.990996837615967, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.6414670944213867, 'logps/chosen': -70.23330688476562, 'logps/rejected': -178.3085479736328, 'logps/ref_chosen': -37.46366882324219, 'logps/ref_rejected': -81.10179138183594, 'logits/chosen': -2.7107484340667725, 'logits/rejected': -2.624185085296631, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.04112689569592476, 'epsilon_dpo/loss_margin_mean': 64.43711853027344, 'epsilon_dpo/beta_margin_mean': 2.6414670944213867, 'epsilon_dpo/beta_margin_std': 2.408226251602173, 'epsilon_dpo/beta_margin_grad_mean': -0.18886899948120117, 'epsilon_dpo/beta_margin_grad_std': 0.20472285151481628, 'kl/beta': 0.04140745848417282, 'kl/avg_steps': 0.6875, 'epoch': 0.19}
|
|||
|
|
19%|█████████████████████▍ | 128/681 [06:03<23:42, 2.57s/it]
19%|█████████████████████▌ | 129/681 [06:06<23:47, 2.59s/it]
{'loss': 0.4566, 'grad_norm': 62.1826057434082, 'learning_rate': 4.88621404556699e-07, 'rewards/chosen': -1.3921856880187988, 'rewards/rejected': -3.942657709121704, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.5504720211029053, 'logps/chosen': -82.27946472167969, 'logps/rejected': -194.88021850585938, 'logps/ref_chosen': -48.18524932861328, 'logps/ref_rejected': -98.1295166015625, 'logits/chosen': -2.753101348876953, 'logits/rejected': -2.694021701812744, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.040820371359586716, 'epsilon_dpo/loss_margin_mean': 62.65648651123047, 'epsilon_dpo/beta_margin_mean': 2.5504720211029053, 'epsilon_dpo/beta_margin_std': 2.0157787799835205, 'epsilon_dpo/beta_margin_grad_mean': -0.16870415210723877, 'epsilon_dpo/beta_margin_grad_std': 0.19453568756580353, 'kl/beta': 0.041124723851680756, 'kl/avg_steps': 0.75, 'epoch': 0.19}
|
|||
|
|
19%|█████████████████████▌ | 129/681 [06:06<23:47, 2.59s/it]
19%|█████████████████████▊ | 130/681 [06:08<23:14, 2.53s/it]
{'loss': 0.3878, 'grad_norm': 67.861572265625, 'learning_rate': 4.882355001067891e-07, 'rewards/chosen': -1.3256336450576782, 'rewards/rejected': -4.240693092346191, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.9150590896606445, 'logps/chosen': -60.728355407714844, 'logps/rejected': -192.62353515625, 'logps/ref_chosen': -28.03460121154785, 'logps/ref_rejected': -87.78228759765625, 'logits/chosen': -2.629741668701172, 'logits/rejected': -2.686960220336914, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.04049098491668701, 'epsilon_dpo/loss_margin_mean': 72.14749145507812, 'epsilon_dpo/beta_margin_mean': 2.9150590896606445, 'epsilon_dpo/beta_margin_std': 2.0989155769348145, 'epsilon_dpo/beta_margin_grad_mean': -0.14500190317630768, 'epsilon_dpo/beta_margin_grad_std': 0.1896047294139862, 'kl/beta': 0.04081858694553375, 'kl/avg_steps': 0.8125, 'epoch': 0.19}
|
|||
|
|
19%|█████████████████████▊ | 130/681 [06:08<23:14, 2.53s/it]
19%|█████████████████████▉ | 131/681 [06:11<23:34, 2.57s/it]
{'loss': 0.4887, 'grad_norm': 55.52901077270508, 'learning_rate': 4.878433179298909e-07, 'rewards/chosen': -1.201073169708252, 'rewards/rejected': -3.551213264465332, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.35014009475708, 'logps/chosen': -64.12167358398438, 'logps/rejected': -182.7357177734375, 'logps/ref_chosen': -34.23711013793945, 'logps/ref_rejected': -94.21966552734375, 'logits/chosen': -2.6353015899658203, 'logits/rejected': -2.799935817718506, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.040177300572395325, 'epsilon_dpo/loss_margin_mean': 58.63149642944336, 'epsilon_dpo/beta_margin_mean': 2.35014009475708, 'epsilon_dpo/beta_margin_std': 1.9053841829299927, 'epsilon_dpo/beta_margin_grad_mean': -0.18838103115558624, 'epsilon_dpo/beta_margin_grad_std': 0.19518207013607025, 'kl/beta': 0.04048960655927658, 'kl/avg_steps': 0.78125, 'epoch': 0.19}
|
|||
|
|
19%|█████████████████████▉ | 131/681 [06:11<23:34, 2.57s/it]
19%|██████████████████████ | 132/681 [06:13<22:52, 2.50s/it]
{'loss': 0.3811, 'grad_norm': 69.0653305053711, 'learning_rate': 4.874448683603694e-07, 'rewards/chosen': -1.5439248085021973, 'rewards/rejected': -4.028234958648682, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.4843099117279053, 'logps/chosen': -86.7735824584961, 'logps/rejected': -194.64205932617188, 'logps/ref_chosen': -48.03201675415039, 'logps/ref_rejected': -93.43745422363281, 'logits/chosen': -2.7254421710968018, 'logits/rejected': -2.769134044647217, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0398343950510025, 'epsilon_dpo/loss_margin_mean': 62.46304702758789, 'epsilon_dpo/beta_margin_mean': 2.4843099117279053, 'epsilon_dpo/beta_margin_std': 1.845669150352478, 'epsilon_dpo/beta_margin_grad_mean': -0.15264593064785004, 'epsilon_dpo/beta_margin_grad_std': 0.15929141640663147, 'kl/beta': 0.04017573595046997, 'kl/avg_steps': 0.859375, 'epoch': 0.19}
|
|||
|
|
19%|██████████████████████ | 132/681 [06:13<22:52, 2.50s/it]
20%|██████████████████████▎ | 133/681 [06:16<22:55, 2.51s/it]
{'loss': 0.3486, 'grad_norm': 58.320655822753906, 'learning_rate': 4.870401618977415e-07, 'rewards/chosen': -1.3570879697799683, 'rewards/rejected': -4.045383930206299, 'rewards/accuracies': 0.984375, 'rewards/margins': 2.688295841217041, 'logps/chosen': -73.2670669555664, 'logps/rejected': -206.1801300048828, 'logps/ref_chosen': -38.89875793457031, 'logps/ref_rejected': -103.64884948730469, 'logits/chosen': -2.6911277770996094, 'logits/rejected': -2.8012609481811523, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.03946392610669136, 'epsilon_dpo/loss_margin_mean': 68.16297149658203, 'epsilon_dpo/beta_margin_mean': 2.68829607963562, 'epsilon_dpo/beta_margin_std': 1.8531904220581055, 'epsilon_dpo/beta_margin_grad_mean': -0.14380493760108948, 'epsilon_dpo/beta_margin_grad_std': 0.15240231156349182, 'kl/beta': 0.03983341529965401, 'kl/avg_steps': 0.9375, 'epoch': 0.2}
|
|||
|
|
20%|██████████████████████▎ | 133/681 [06:16<22:55, 2.51s/it]
20%|██████████████████████▍ | 134/681 [06:18<22:33, 2.47s/it]
{'loss': 0.3876, 'grad_norm': 53.25428009033203, 'learning_rate': 4.866292092063986e-07, 'rewards/chosen': -1.346604824066162, 'rewards/rejected': -3.877628803253174, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.531024217605591, 'logps/chosen': -71.48147583007812, 'logps/rejected': -192.48870849609375, 'logps/ref_chosen': -37.08332061767578, 'logps/ref_rejected': -93.31185150146484, 'logits/chosen': -2.665362596511841, 'logits/rejected': -2.729599952697754, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.03912205249071121, 'epsilon_dpo/loss_margin_mean': 64.7786865234375, 'epsilon_dpo/beta_margin_mean': 2.531024217605591, 'epsilon_dpo/beta_margin_std': 1.835780143737793, 'epsilon_dpo/beta_margin_grad_mean': -0.15723571181297302, 'epsilon_dpo/beta_margin_grad_std': 0.16265857219696045, 'kl/beta': 0.03946344554424286, 'kl/avg_steps': 0.875, 'epoch': 0.2}
|
|||
|
|
20%|██████████████████████▍ | 134/681 [06:18<22:33, 2.47s/it]
20%|██████████████████████▌ | 135/681 [06:21<22:31, 2.48s/it]
{'loss': 0.3559, 'grad_norm': 58.53507995605469, 'learning_rate': 4.862120211153265e-07, 'rewards/chosen': -1.5515646934509277, 'rewards/rejected': -4.2581939697265625, 'rewards/accuracies': 0.984375, 'rewards/margins': 2.7066292762756348, 'logps/chosen': -68.14396667480469, 'logps/rejected': -230.93429565429688, 'logps/ref_chosen': -28.130008697509766, 'logps/ref_rejected': -121.04113006591797, 'logits/chosen': -2.4796104431152344, 'logits/rejected': -2.7320666313171387, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.03875825181603432, 'epsilon_dpo/loss_margin_mean': 69.87920379638672, 'epsilon_dpo/beta_margin_mean': 2.7066292762756348, 'epsilon_dpo/beta_margin_std': 2.084244966506958, 'epsilon_dpo/beta_margin_grad_mean': -0.14636199176311493, 'epsilon_dpo/beta_margin_grad_std': 0.1549866497516632, 'kl/beta': 0.03912113606929779, 'kl/avg_steps': 0.9375, 'epoch': 0.2}
|
|||
|
|
20%|██████████████████████▌ | 135/681 [06:21<22:31, 2.48s/it]
20%|██████████████████████▊ | 136/681 [06:23<22:45, 2.50s/it]
{'loss': 0.4334, 'grad_norm': 71.29747009277344, 'learning_rate': 4.857886086178193e-07, 'rewards/chosen': -1.5163445472717285, 'rewards/rejected': -4.36431884765625, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.8479745388031006, 'logps/chosen': -80.63860321044922, 'logps/rejected': -216.74537658691406, 'logps/ref_chosen': -41.21574401855469, 'logps/ref_rejected': -103.0965805053711, 'logits/chosen': -2.642530918121338, 'logits/rejected': -2.726252794265747, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.03843460604548454, 'epsilon_dpo/loss_margin_mean': 74.22594451904297, 'epsilon_dpo/beta_margin_mean': 2.8479745388031006, 'epsilon_dpo/beta_margin_std': 2.358449935913086, 'epsilon_dpo/beta_margin_grad_mean': -0.1646306961774826, 'epsilon_dpo/beta_margin_grad_std': 0.19557887315750122, 'kl/beta': 0.03875778242945671, 'kl/avg_steps': 0.84375, 'epoch': 0.2}
|
|||
|
|
20%|██████████████████████▊ | 136/681 [06:23<22:45, 2.50s/it]
20%|██████████████████████▉ | 137/681 [06:26<22:31, 2.48s/it]
{'loss': 0.3075, 'grad_norm': 50.14603805541992, 'learning_rate': 4.853589828711902e-07, 'rewards/chosen': -1.6396162509918213, 'rewards/rejected': -5.06685733795166, 'rewards/accuracies': 0.96875, 'rewards/margins': 3.427241325378418, 'logps/chosen': -75.71855163574219, 'logps/rejected': -252.65704345703125, 'logps/ref_chosen': -32.656131744384766, 'logps/ref_rejected': -119.52920532226562, 'logits/chosen': -2.384611129760742, 'logits/rejected': -2.656006336212158, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.03807699307799339, 'epsilon_dpo/loss_margin_mean': 90.0654296875, 'epsilon_dpo/beta_margin_mean': 3.427241325378418, 'epsilon_dpo/beta_margin_std': 2.608391523361206, 'epsilon_dpo/beta_margin_grad_mean': -0.125637486577034, 'epsilon_dpo/beta_margin_grad_std': 0.15655429661273956, 'kl/beta': 0.0384334996342659, 'kl/avg_steps': 0.9375, 'epoch': 0.2}
|
|||
|
|
20%|██████████████████████▉ | 137/681 [06:26<22:31, 2.48s/it]
20%|███████████████████████ | 138/681 [06:28<22:18, 2.47s/it]
{'loss': 0.3421, 'grad_norm': 64.70891571044922, 'learning_rate': 4.849231551964771e-07, 'rewards/chosen': -1.8868746757507324, 'rewards/rejected': -5.076008319854736, 'rewards/accuracies': 0.9375, 'rewards/margins': 3.189133644104004, 'logps/chosen': -81.6050796508789, 'logps/rejected': -234.10885620117188, 'logps/ref_chosen': -31.68457794189453, 'logps/ref_rejected': -99.58272552490234, 'logits/chosen': -2.3941986560821533, 'logits/rejected': -2.439361095428467, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.03774713724851608, 'epsilon_dpo/loss_margin_mean': 84.6056137084961, 'epsilon_dpo/beta_margin_mean': 3.189133644104004, 'epsilon_dpo/beta_margin_std': 2.1990420818328857, 'epsilon_dpo/beta_margin_grad_mean': -0.13305923342704773, 'epsilon_dpo/beta_margin_grad_std': 0.17672809958457947, 'kl/beta': 0.038076531141996384, 'kl/avg_steps': 0.875, 'epoch': 0.2}
|
|||
|
|
20%|███████████████████████ | 138/681 [06:28<22:18, 2.47s/it]
20%|███████████████████████▎ | 139/681 [06:31<22:18, 2.47s/it]
{'loss': 0.3096, 'grad_norm': 64.89179992675781, 'learning_rate': 4.844811370781446e-07, 'rewards/chosen': -1.656851053237915, 'rewards/rejected': -5.11447811126709, 'rewards/accuracies': 0.953125, 'rewards/margins': 3.4576268196105957, 'logps/chosen': -74.70332336425781, 'logps/rejected': -222.0337371826172, 'logps/ref_chosen': -30.504302978515625, 'logps/ref_rejected': -85.29287719726562, 'logits/chosen': -2.3702569007873535, 'logits/rejected': -2.3330612182617188, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.03741971403360367, 'epsilon_dpo/loss_margin_mean': 92.54183959960938, 'epsilon_dpo/beta_margin_mean': 3.4576268196105957, 'epsilon_dpo/beta_margin_std': 2.2132863998413086, 'epsilon_dpo/beta_margin_grad_mean': -0.10873990505933762, 'epsilon_dpo/beta_margin_grad_std': 0.18296301364898682, 'kl/beta': 0.03774625062942505, 'kl/avg_steps': 0.875, 'epoch': 0.2}
|
|||
|
|
20%|███████████████████████▎ | 139/681 [06:31<22:18, 2.47s/it]
21%|███████████████████████▍ | 140/681 [06:33<22:07, 2.45s/it]
{'loss': 0.3733, 'grad_norm': 64.97393035888672, 'learning_rate': 4.840329401637809e-07, 'rewards/chosen': -1.8963526487350464, 'rewards/rejected': -4.897425651550293, 'rewards/accuracies': 0.9375, 'rewards/margins': 3.001072883605957, 'logps/chosen': -87.36968994140625, 'logps/rejected': -221.3934326171875, 'logps/ref_chosen': -36.32595443725586, 'logps/ref_rejected': -89.32574462890625, 'logits/chosen': -2.463718891143799, 'logits/rejected': -2.454908609390259, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.037118520587682724, 'epsilon_dpo/loss_margin_mean': 81.0239486694336, 'epsilon_dpo/beta_margin_mean': 3.001073122024536, 'epsilon_dpo/beta_margin_std': 2.1810362339019775, 'epsilon_dpo/beta_margin_grad_mean': -0.14209268987178802, 'epsilon_dpo/beta_margin_grad_std': 0.18320953845977783, 'kl/beta': 0.03741883859038353, 'kl/avg_steps': 0.8125, 'epoch': 0.21}
|
|||
|
|
21%|███████████████████████▍ | 140/681 [06:33<22:07, 2.45s/it]
21%|███████████████████████▌ | 141/681 [06:36<22:23, 2.49s/it]
{'loss': 0.3351, 'grad_norm': 73.34292602539062, 'learning_rate': 4.83578576263792e-07, 'rewards/chosen': -1.991283655166626, 'rewards/rejected': -5.155275344848633, 'rewards/accuracies': 0.9375, 'rewards/margins': 3.1639914512634277, 'logps/chosen': -108.33308410644531, 'logps/rejected': -245.32533264160156, 'logps/ref_chosen': -54.31216049194336, 'logps/ref_rejected': -105.19206237792969, 'logits/chosen': -2.636939287185669, 'logits/rejected': -2.580254316329956, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.03681936860084534, 'epsilon_dpo/loss_margin_mean': 86.11234283447266, 'epsilon_dpo/beta_margin_mean': 3.1639914512634277, 'epsilon_dpo/beta_margin_std': 2.328209638595581, 'epsilon_dpo/beta_margin_grad_mean': -0.12768100202083588, 'epsilon_dpo/beta_margin_grad_std': 0.1734439581632614, 'kl/beta': 0.037117261439561844, 'kl/avg_steps': 0.8125, 'epoch': 0.21}
|
|||
|
|
21%|███████████████████████▌ | 141/681 [06:36<22:23, 2.49s/it]
21%|███████████████████████▊ | 142/681 [06:38<22:55, 2.55s/it]
{'loss': 0.3654, 'grad_norm': 72.56526947021484, 'learning_rate': 4.83118057351089e-07, 'rewards/chosen': -1.9631787538528442, 'rewards/rejected': -5.146324157714844, 'rewards/accuracies': 0.9375, 'rewards/margins': 3.183145523071289, 'logps/chosen': -91.45923614501953, 'logps/rejected': -241.49392700195312, 'logps/ref_chosen': -37.70738220214844, 'logps/ref_rejected': -100.41778564453125, 'logits/chosen': -2.3550641536712646, 'logits/rejected': -2.433764934539795, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.03649960458278656, 'epsilon_dpo/loss_margin_mean': 87.32428741455078, 'epsilon_dpo/beta_margin_mean': 3.183145761489868, 'epsilon_dpo/beta_margin_std': 2.435292959213257, 'epsilon_dpo/beta_margin_grad_mean': -0.14174862205982208, 'epsilon_dpo/beta_margin_grad_std': 0.17707964777946472, 'kl/beta': 0.036818113178014755, 'kl/avg_steps': 0.875, 'epoch': 0.21}
|
|||
|
|
21%|███████████████████████▊ | 142/681 [06:38<22:55, 2.55s/it]
21%|███████████████████████▉ | 143/681 [06:41<22:38, 2.53s/it]
{'loss': 0.5517, 'grad_norm': 89.1434555053711, 'learning_rate': 4.826513955607734e-07, 'rewards/chosen': -1.7896441221237183, 'rewards/rejected': -4.518563270568848, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.72891902923584, 'logps/chosen': -89.65013122558594, 'logps/rejected': -213.12933349609375, 'logps/ref_chosen': -40.36812210083008, 'logps/ref_rejected': -88.28787994384766, 'logits/chosen': -2.482374668121338, 'logits/rejected': -2.3714542388916016, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.03624003753066063, 'epsilon_dpo/loss_margin_mean': 75.5594482421875, 'epsilon_dpo/beta_margin_mean': 2.72891902923584, 'epsilon_dpo/beta_margin_std': 2.580869674682617, 'epsilon_dpo/beta_margin_grad_mean': -0.1898718923330307, 'epsilon_dpo/beta_margin_grad_std': 0.2213219702243805, 'kl/beta': 0.036498747766017914, 'kl/avg_steps': 0.71875, 'epoch': 0.21}
|
|||
|
|
21%|███████████████████████▉ | 143/681 [06:41<22:38, 2.53s/it]
21%|████████████████████████ | 144/681 [06:43<23:12, 2.59s/it]
{'loss': 0.4112, 'grad_norm': 74.61936950683594, 'learning_rate': 4.821786031898176e-07, 'rewards/chosen': -1.765446662902832, 'rewards/rejected': -4.696205139160156, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.930758476257324, 'logps/chosen': -90.16825103759766, 'logps/rejected': -218.21946716308594, 'logps/ref_chosen': -41.09404373168945, 'logps/ref_rejected': -87.43101501464844, 'logits/chosen': -2.5220513343811035, 'logits/rejected': -2.4428181648254395, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.03593612089753151, 'epsilon_dpo/loss_margin_mean': 81.71424102783203, 'epsilon_dpo/beta_margin_mean': 2.930758237838745, 'epsilon_dpo/beta_margin_std': 2.2233402729034424, 'epsilon_dpo/beta_margin_grad_mean': -0.14966890215873718, 'epsilon_dpo/beta_margin_grad_std': 0.1911356896162033, 'kl/beta': 0.03623828664422035, 'kl/avg_steps': 0.84375, 'epoch': 0.21}
|
|||
|
|
21%|████████████████████████ | 144/681 [06:43<23:12, 2.59s/it]
21%|████████████████████████▎ | 145/681 [06:46<22:48, 2.55s/it]
{'loss': 0.2806, 'grad_norm': 62.392303466796875, 'learning_rate': 4.816996926967401e-07, 'rewards/chosen': -1.800328016281128, 'rewards/rejected': -5.022313117980957, 'rewards/accuracies': 0.96875, 'rewards/margins': 3.221985340118408, 'logps/chosen': -86.23258972167969, 'logps/rejected': -225.2410888671875, 'logps/ref_chosen': -35.7225341796875, 'logps/ref_rejected': -84.17701721191406, 'logits/chosen': -2.4150798320770264, 'logits/rejected': -2.3624231815338135, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.035612985491752625, 'epsilon_dpo/loss_margin_mean': 90.55402374267578, 'epsilon_dpo/beta_margin_mean': 3.221985101699829, 'epsilon_dpo/beta_margin_std': 2.2744228839874268, 'epsilon_dpo/beta_margin_grad_mean': -0.11719842255115509, 'epsilon_dpo/beta_margin_grad_std': 0.13962174952030182, 'kl/beta': 0.03593508526682854, 'kl/avg_steps': 0.90625, 'epoch': 0.21}
|
|||
|
|
21%|████████████████████████▎ | 145/681 [06:46<22:48, 2.55s/it]
21%|████████████████████████▍ | 146/681 [06:48<22:35, 2.53s/it]
{'loss': 0.3739, 'grad_norm': 61.81754684448242, 'learning_rate': 4.812146767012779e-07, 'rewards/chosen': -2.0904293060302734, 'rewards/rejected': -5.120657444000244, 'rewards/accuracies': 0.953125, 'rewards/margins': 3.0302281379699707, 'logps/chosen': -100.90562438964844, 'logps/rejected': -233.08261108398438, 'logps/ref_chosen': -41.776546478271484, 'logps/ref_rejected': -87.99478149414062, 'logits/chosen': -2.4440078735351562, 'logits/rejected': -2.263402223587036, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.03531539812684059, 'epsilon_dpo/loss_margin_mean': 85.95875549316406, 'epsilon_dpo/beta_margin_mean': 3.0302278995513916, 'epsilon_dpo/beta_margin_std': 2.1277804374694824, 'epsilon_dpo/beta_margin_grad_mean': -0.1365576684474945, 'epsilon_dpo/beta_margin_grad_std': 0.18428391218185425, 'kl/beta': 0.03561234846711159, 'kl/avg_steps': 0.84375, 'epoch': 0.21}
|
|||
|
|
21%|████████████████████████▍ | 146/681 [06:48<22:35, 2.53s/it]
22%|████████████████████████▌ | 147/681 [06:51<22:07, 2.49s/it]
{'loss': 0.4282, 'grad_norm': 94.37625122070312, 'learning_rate': 4.807235679840536e-07, 'rewards/chosen': -1.9178650379180908, 'rewards/rejected': -5.034238815307617, 'rewards/accuracies': 0.921875, 'rewards/margins': 3.1163740158081055, 'logps/chosen': -89.63932800292969, 'logps/rejected': -220.49072265625, 'logps/ref_chosen': -35.02455139160156, 'logps/ref_rejected': -76.69776153564453, 'logits/chosen': -2.4340085983276367, 'logits/rejected': -2.291194200515747, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.03503095358610153, 'epsilon_dpo/loss_margin_mean': 89.17818450927734, 'epsilon_dpo/beta_margin_mean': 3.1163737773895264, 'epsilon_dpo/beta_margin_std': 2.363018274307251, 'epsilon_dpo/beta_margin_grad_mean': -0.14339102804660797, 'epsilon_dpo/beta_margin_grad_std': 0.21028229594230652, 'kl/beta': 0.03531438112258911, 'kl/avg_steps': 0.8125, 'epoch': 0.22}
|
|||
|
|
22%|████████████████████████▌ | 147/681 [06:51<22:07, 2.49s/it]
22%|████████████████████████▊ | 148/681 [06:53<22:19, 2.51s/it]
{'loss': 0.4897, 'grad_norm': 83.76101684570312, 'learning_rate': 4.802263794862384e-07, 'rewards/chosen': -1.9978649616241455, 'rewards/rejected': -4.399646759033203, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.4017815589904785, 'logps/chosen': -100.62034606933594, 'logps/rejected': -235.7743682861328, 'logps/ref_chosen': -43.158294677734375, 'logps/ref_rejected': -109.0255126953125, 'logits/chosen': -2.529165744781494, 'logits/rejected': -2.569955825805664, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.03473767638206482, 'epsilon_dpo/loss_margin_mean': 69.28678894042969, 'epsilon_dpo/beta_margin_mean': 2.4017815589904785, 'epsilon_dpo/beta_margin_std': 2.13431715965271, 'epsilon_dpo/beta_margin_grad_mean': -0.18217557668685913, 'epsilon_dpo/beta_margin_grad_std': 0.1949581652879715, 'kl/beta': 0.03502976521849632, 'kl/avg_steps': 0.84375, 'epoch': 0.22}
|
|||
|
|
22%|████████████████████████▊ | 148/681 [06:53<22:19, 2.51s/it]
22%|████████████████████████▉ | 149/681 [06:56<22:33, 2.54s/it]
{'loss': 0.383, 'grad_norm': 69.50299835205078, 'learning_rate': 4.797231243092118e-07, 'rewards/chosen': -2.014852285385132, 'rewards/rejected': -4.940248489379883, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.92539644241333, 'logps/chosen': -93.2674331665039, 'logps/rejected': -244.69888305664062, 'logps/ref_chosen': -34.801856994628906, 'logps/ref_rejected': -101.17015075683594, 'logits/chosen': -2.407914638519287, 'logits/rejected': -2.4749755859375, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0344361737370491, 'epsilon_dpo/loss_margin_mean': 85.06315612792969, 'epsilon_dpo/beta_margin_mean': 2.92539644241333, 'epsilon_dpo/beta_margin_std': 2.3435420989990234, 'epsilon_dpo/beta_margin_grad_mean': -0.15145255625247955, 'epsilon_dpo/beta_margin_grad_std': 0.1748467981815338, 'kl/beta': 0.03473667427897453, 'kl/avg_steps': 0.875, 'epoch': 0.22}
|
|||
|
|
22%|████████████████████████▉ | 149/681 [06:56<22:33, 2.54s/it]
22%|█████████████████████████ | 150/681 [06:58<22:26, 2.53s/it]
{'loss': 0.3576, 'grad_norm': 67.23019409179688, 'learning_rate': 4.792138157142157e-07, 'rewards/chosen': -1.8020522594451904, 'rewards/rejected': -4.602282524108887, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.800230026245117, 'logps/chosen': -79.86786651611328, 'logps/rejected': -223.50430297851562, 'logps/ref_chosen': -27.168495178222656, 'logps/ref_rejected': -88.6546859741211, 'logits/chosen': -2.3103485107421875, 'logits/rejected': -2.399022102355957, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.034148234874010086, 'epsilon_dpo/loss_margin_mean': 82.1502456665039, 'epsilon_dpo/beta_margin_mean': 2.8002302646636963, 'epsilon_dpo/beta_margin_std': 1.951874852180481, 'epsilon_dpo/beta_margin_grad_mean': -0.13967867195606232, 'epsilon_dpo/beta_margin_grad_std': 0.1724717617034912, 'kl/beta': 0.03443536534905434, 'kl/avg_steps': 0.84375, 'epoch': 0.22}
|
|||
|
|
22%|█████████████████████████ | 150/681 [06:59<22:26, 2.53s/it]
22%|█████████████████████████▎ | 151/681 [07:01<22:34, 2.56s/it]
{'loss': 0.2918, 'grad_norm': 56.315635681152344, 'learning_rate': 4.786984671220053e-07, 'rewards/chosen': -1.8228228092193604, 'rewards/rejected': -4.796318054199219, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.9734950065612793, 'logps/chosen': -98.35040283203125, 'logps/rejected': -249.35023498535156, 'logps/ref_chosen': -44.517967224121094, 'logps/ref_rejected': -107.56175994873047, 'logits/chosen': -2.5218310356140137, 'logits/rejected': -2.627288341522217, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.033841170370578766, 'epsilon_dpo/loss_margin_mean': 87.95604705810547, 'epsilon_dpo/beta_margin_mean': 2.9734950065612793, 'epsilon_dpo/beta_margin_std': 1.9034627676010132, 'epsilon_dpo/beta_margin_grad_mean': -0.12026865780353546, 'epsilon_dpo/beta_margin_grad_std': 0.14584773778915405, 'kl/beta': 0.034147247672080994, 'kl/avg_steps': 0.90625, 'epoch': 0.22}
|
|||
|
|
22%|█████████████████████████▎ | 151/681 [07:01<22:34, 2.56s/it]
22%|█████████████████████████▍ | 152/681 [07:03<22:08, 2.51s/it]
{'loss': 0.3243, 'grad_norm': 68.73706817626953, 'learning_rate': 4.78177092112495e-07, 'rewards/chosen': -1.9261062145233154, 'rewards/rejected': -5.093712329864502, 'rewards/accuracies': 0.9375, 'rewards/margins': 3.1676058769226074, 'logps/chosen': -102.739501953125, 'logps/rejected': -264.4252624511719, 'logps/ref_chosen': -45.36680603027344, 'logps/ref_rejected': -112.48715209960938, 'logits/chosen': -2.4150760173797607, 'logits/rejected': -2.4403088092803955, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.033547814935445786, 'epsilon_dpo/loss_margin_mean': 94.56542205810547, 'epsilon_dpo/beta_margin_mean': 3.1676061153411865, 'epsilon_dpo/beta_margin_std': 2.1665868759155273, 'epsilon_dpo/beta_margin_grad_mean': -0.12259558588266373, 'epsilon_dpo/beta_margin_grad_std': 0.1732029914855957, 'kl/beta': 0.033840566873550415, 'kl/avg_steps': 0.875, 'epoch': 0.22}
|
|||
|
|
22%|█████████████████████████▍ | 152/681 [07:04<22:08, 2.51s/it]
22%|█████████████████████████▌ | 153/681 [07:06<21:55, 2.49s/it]
{'loss': 0.468, 'grad_norm': 84.02366638183594, 'learning_rate': 4.776497044244016e-07, 'rewards/chosen': -1.9907963275909424, 'rewards/rejected': -4.86927604675293, 'rewards/accuracies': 0.84375, 'rewards/margins': 2.878479480743408, 'logps/chosen': -99.41288757324219, 'logps/rejected': -238.06539916992188, 'logps/ref_chosen': -39.70823669433594, 'logps/ref_rejected': -91.66893768310547, 'logits/chosen': -2.4134445190429688, 'logits/rejected': -2.408932685852051, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.033309243619441986, 'epsilon_dpo/loss_margin_mean': 86.69181060791016, 'epsilon_dpo/beta_margin_mean': 2.878479480743408, 'epsilon_dpo/beta_margin_std': 2.4571328163146973, 'epsilon_dpo/beta_margin_grad_mean': -0.17181988060474396, 'epsilon_dpo/beta_margin_grad_std': 0.2117866724729538, 'kl/beta': 0.03354702889919281, 'kl/avg_steps': 0.71875, 'epoch': 0.22}
|
|||
|
|
22%|█████████████████████████▌ | 153/681 [07:06<21:55, 2.49s/it]
23%|█████████████████████████▊ | 154/681 [07:09<22:26, 2.56s/it]
{'loss': 0.4488, 'grad_norm': 101.56253051757812, 'learning_rate': 4.771163179548808e-07, 'rewards/chosen': -1.834795594215393, 'rewards/rejected': -5.068642616271973, 'rewards/accuracies': 0.859375, 'rewards/margins': 3.233847141265869, 'logps/chosen': -91.09129333496094, 'logps/rejected': -264.642333984375, 'logps/ref_chosen': -35.708839416503906, 'logps/ref_rejected': -111.15846252441406, 'logits/chosen': -2.3385679721832275, 'logits/rejected': -2.41574764251709, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.033061131834983826, 'epsilon_dpo/loss_margin_mean': 98.10140991210938, 'epsilon_dpo/beta_margin_mean': 3.233847141265869, 'epsilon_dpo/beta_margin_std': 2.5847911834716797, 'epsilon_dpo/beta_margin_grad_mean': -0.15857023000717163, 'epsilon_dpo/beta_margin_grad_std': 0.21813619136810303, 'kl/beta': 0.03330763056874275, 'kl/avg_steps': 0.75, 'epoch': 0.23}
|
|||
|
|
23%|█████████████████████████▊ | 154/681 [07:09<22:26, 2.56s/it]
23%|█████████████████████████▉ | 155/681 [07:11<22:25, 2.56s/it]
{'loss': 0.384, 'grad_norm': 63.52682876586914, 'learning_rate': 4.7657694675916247e-07, 'rewards/chosen': -1.9715728759765625, 'rewards/rejected': -4.697756767272949, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.7261838912963867, 'logps/chosen': -102.16056823730469, 'logps/rejected': -227.42694091796875, 'logps/ref_chosen': -42.04505157470703, 'logps/ref_rejected': -84.00552368164062, 'logits/chosen': -2.438190460205078, 'logits/rejected': -2.357145071029663, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.03278402239084244, 'epsilon_dpo/loss_margin_mean': 83.30589294433594, 'epsilon_dpo/beta_margin_mean': 2.7261838912963867, 'epsilon_dpo/beta_margin_std': 1.8952271938323975, 'epsilon_dpo/beta_margin_grad_mean': -0.1480940580368042, 'epsilon_dpo/beta_margin_grad_std': 0.18447977304458618, 'kl/beta': 0.033059682697057724, 'kl/avg_steps': 0.84375, 'epoch': 0.23}
|
|||
|
|
23%|█████████████████████████▉ | 155/681 [07:11<22:25, 2.56s/it]
23%|██████████████████████████ | 156/681 [07:14<22:30, 2.57s/it]
{'loss': 0.5071, 'grad_norm': 146.7109375, 'learning_rate': 4.7603160505017893e-07, 'rewards/chosen': -2.1660971641540527, 'rewards/rejected': -4.9945268630981445, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.828429698944092, 'logps/chosen': -115.40594482421875, 'logps/rejected': -237.8277130126953, 'logps/ref_chosen': -49.038185119628906, 'logps/ref_rejected': -84.26383209228516, 'logits/chosen': -2.4288077354431152, 'logits/rejected': -2.2443032264709473, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.03256094828248024, 'epsilon_dpo/loss_margin_mean': 87.19612121582031, 'epsilon_dpo/beta_margin_mean': 2.8284294605255127, 'epsilon_dpo/beta_margin_std': 2.360419750213623, 'epsilon_dpo/beta_margin_grad_mean': -0.16920383274555206, 'epsilon_dpo/beta_margin_grad_std': 0.22653749585151672, 'kl/beta': 0.03278307616710663, 'kl/avg_steps': 0.6875, 'epoch': 0.23}
|
|||
|
|
23%|██████████████████████████ | 156/681 [07:14<22:30, 2.57s/it]
23%|██████████████████████████▎ | 157/681 [07:16<21:26, 2.45s/it]
{'loss': 0.3039, 'grad_norm': 74.27545928955078, 'learning_rate': 4.7548030719819154e-07, 'rewards/chosen': -1.7866594791412354, 'rewards/rejected': -5.353865623474121, 'rewards/accuracies': 0.953125, 'rewards/margins': 3.567206382751465, 'logps/chosen': -85.81341552734375, 'logps/rejected': -280.2191162109375, 'logps/ref_chosen': -30.534713745117188, 'logps/ref_rejected': -114.2703628540039, 'logits/chosen': -2.416853904724121, 'logits/rejected': -2.5221433639526367, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.03226739168167114, 'epsilon_dpo/loss_margin_mean': 110.67003631591797, 'epsilon_dpo/beta_margin_mean': 3.567206382751465, 'epsilon_dpo/beta_margin_std': 2.317911386489868, 'epsilon_dpo/beta_margin_grad_mean': -0.11788501590490341, 'epsilon_dpo/beta_margin_grad_std': 0.1734883189201355, 'kl/beta': 0.03255923092365265, 'kl/avg_steps': 0.90625, 'epoch': 0.23}
|
|||
|
|
23%|██████████████████████████▎ | 157/681 [07:16<21:26, 2.45s/it]
23%|██████████████████████████▍ | 158/681 [07:19<21:35, 2.48s/it]
{'loss': 0.2905, 'grad_norm': 52.514793395996094, 'learning_rate': 4.7492306773041136e-07, 'rewards/chosen': -1.5289018154144287, 'rewards/rejected': -4.84739875793457, 'rewards/accuracies': 0.953125, 'rewards/margins': 3.3184967041015625, 'logps/chosen': -77.87710571289062, 'logps/rejected': -271.2097473144531, 'logps/ref_chosen': -30.166948318481445, 'logps/ref_rejected': -119.61189270019531, 'logits/chosen': -2.434067726135254, 'logits/rejected': -2.5916080474853516, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.03198767825961113, 'epsilon_dpo/loss_margin_mean': 103.88768768310547, 'epsilon_dpo/beta_margin_mean': 3.3184967041015625, 'epsilon_dpo/beta_margin_std': 2.208585739135742, 'epsilon_dpo/beta_margin_grad_mean': -0.11316555738449097, 'epsilon_dpo/beta_margin_grad_std': 0.1629510372877121, 'kl/beta': 0.03226681426167488, 'kl/avg_steps': 0.875, 'epoch': 0.23}
|
|||
|
|
23%|██████████████████████████▍ | 158/681 [07:19<21:35, 2.48s/it]
23%|██████████████████████████▌ | 159/681 [07:21<21:50, 2.51s/it]
{'loss': 0.4372, 'grad_norm': 72.99739074707031, 'learning_rate': 4.743599013306165e-07, 'rewards/chosen': -1.6737987995147705, 'rewards/rejected': -4.784216403961182, 'rewards/accuracies': 0.90625, 'rewards/margins': 3.110417366027832, 'logps/chosen': -96.61356353759766, 'logps/rejected': -247.14047241210938, 'logps/ref_chosen': -43.95257568359375, 'logps/ref_rejected': -96.23291015625, 'logits/chosen': -2.645659923553467, 'logits/rejected': -2.5504307746887207, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.03173020854592323, 'epsilon_dpo/loss_margin_mean': 98.24658203125, 'epsilon_dpo/beta_margin_mean': 3.110417604446411, 'epsilon_dpo/beta_margin_std': 2.455885410308838, 'epsilon_dpo/beta_margin_grad_mean': -0.1506548821926117, 'epsilon_dpo/beta_margin_grad_std': 0.20841404795646667, 'kl/beta': 0.03198692947626114, 'kl/avg_steps': 0.8125, 'epoch': 0.23}
|
|||
|
|
23%|██████████████████████████▌ | 159/681 [07:21<21:50, 2.51s/it]
23%|██████████████████████████▊ | 160/681 [07:23<21:22, 2.46s/it]
{'loss': 0.5131, 'grad_norm': 85.227294921875, 'learning_rate': 4.737908228387656e-07, 'rewards/chosen': -1.6889050006866455, 'rewards/rejected': -4.517616271972656, 'rewards/accuracies': 0.875, 'rewards/margins': 2.82871150970459, 'logps/chosen': -97.44248962402344, 'logps/rejected': -247.85491943359375, 'logps/ref_chosen': -43.95906066894531, 'logps/ref_rejected': -104.28060913085938, 'logits/chosen': -2.586042881011963, 'logits/rejected': -2.5926198959350586, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.03151414170861244, 'epsilon_dpo/loss_margin_mean': 90.09088134765625, 'epsilon_dpo/beta_margin_mean': 2.8287112712860107, 'epsilon_dpo/beta_margin_std': 2.4387524127960205, 'epsilon_dpo/beta_margin_grad_mean': -0.17995795607566833, 'epsilon_dpo/beta_margin_grad_std': 0.22594332695007324, 'kl/beta': 0.03172912821173668, 'kl/avg_steps': 0.6875, 'epoch': 0.23}
|
|||
|
|
23%|██████████████████████████▊ | 160/681 [07:23<21:22, 2.46s/it]
24%|██████████████████████████▉ | 161/681 [07:26<20:51, 2.41s/it]
{'loss': 0.4746, 'grad_norm': 69.41145324707031, 'learning_rate': 4.7321584725060594e-07, 'rewards/chosen': -1.5938321352005005, 'rewards/rejected': -4.561304092407227, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.9674715995788574, 'logps/chosen': -81.56330108642578, 'logps/rejected': -234.62916564941406, 'logps/ref_chosen': -30.6236572265625, 'logps/ref_rejected': -88.5151138305664, 'logits/chosen': -2.514247417449951, 'logits/rejected': -2.5099971294403076, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.031249718740582466, 'epsilon_dpo/loss_margin_mean': 95.17440795898438, 'epsilon_dpo/beta_margin_mean': 2.9674715995788574, 'epsilon_dpo/beta_margin_std': 2.1990880966186523, 'epsilon_dpo/beta_margin_grad_mean': -0.14812429249286652, 'epsilon_dpo/beta_margin_grad_std': 0.21790093183517456, 'kl/beta': 0.03151248022913933, 'kl/avg_steps': 0.84375, 'epoch': 0.24}
|
|||
|
|
24%|██████████████████████████▉ | 161/681 [07:26<20:51, 2.41s/it]
24%|███████████████████████████ | 162/681 [07:28<21:24, 2.47s/it]
{'loss': 0.3757, 'grad_norm': 68.47539520263672, 'learning_rate': 4.7263498971727905e-07, 'rewards/chosen': -1.514453411102295, 'rewards/rejected': -4.409099578857422, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.894646167755127, 'logps/chosen': -85.86666870117188, 'logps/rejected': -236.87969970703125, 'logps/ref_chosen': -37.11417007446289, 'logps/ref_rejected': -94.5440673828125, 'logits/chosen': -2.5756616592407227, 'logits/rejected': -2.518211603164673, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.030998021364212036, 'epsilon_dpo/loss_margin_mean': 93.58312225341797, 'epsilon_dpo/beta_margin_mean': 2.894646406173706, 'epsilon_dpo/beta_margin_std': 2.0557403564453125, 'epsilon_dpo/beta_margin_grad_mean': -0.14223085343837738, 'epsilon_dpo/beta_margin_grad_std': 0.18433333933353424, 'kl/beta': 0.031248819082975388, 'kl/avg_steps': 0.8125, 'epoch': 0.24}
|
|||
|
|
24%|███████████████████████████ | 162/681 [07:28<21:24, 2.47s/it]
24%|███████████████████████████▎ | 163/681 [07:31<20:56, 2.43s/it]
{'loss': 0.4252, 'grad_norm': 56.615814208984375, 'learning_rate': 4.720482655449212e-07, 'rewards/chosen': -1.3745607137680054, 'rewards/rejected': -4.175376892089844, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.800816535949707, 'logps/chosen': -75.5699234008789, 'logps/rejected': -217.57525634765625, 'logps/ref_chosen': -30.90496063232422, 'logps/ref_rejected': -81.61336517333984, 'logits/chosen': -2.5021204948425293, 'logits/rejected': -2.4259862899780273, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.030757879838347435, 'epsilon_dpo/loss_margin_mean': 91.29692077636719, 'epsilon_dpo/beta_margin_mean': 2.800816535949707, 'epsilon_dpo/beta_margin_std': 1.9919666051864624, 'epsilon_dpo/beta_margin_grad_mean': -0.15289483964443207, 'epsilon_dpo/beta_margin_grad_std': 0.209597647190094, 'kl/beta': 0.03099696896970272, 'kl/avg_steps': 0.78125, 'epoch': 0.24}
|
|||
|
|
24%|███████████████████████████▎ | 163/681 [07:31<20:56, 2.43s/it]
24%|███████████████████████████▍ | 164/681 [07:33<20:42, 2.40s/it]
{'loss': 0.3178, 'grad_norm': 59.15823745727539, 'learning_rate': 4.714556901942599e-07, 'rewards/chosen': -1.5812957286834717, 'rewards/rejected': -4.190255165100098, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.6089587211608887, 'logps/chosen': -84.63601684570312, 'logps/rejected': -224.88031005859375, 'logps/ref_chosen': -32.731651306152344, 'logps/ref_rejected': -87.29302215576172, 'logits/chosen': -2.5339465141296387, 'logits/rejected': -2.4976041316986084, 'kl/p_epsilon_steps': 0.984375, 'kl/n_epsilon_steps': 0.015625, 'epsilon_dpo/beta': 0.03046177327632904, 'epsilon_dpo/loss_margin_mean': 85.68292999267578, 'epsilon_dpo/beta_margin_mean': 2.6089589595794678, 'epsilon_dpo/beta_margin_std': 1.5135873556137085, 'epsilon_dpo/beta_margin_grad_mean': -0.12999504804611206, 'epsilon_dpo/beta_margin_grad_std': 0.14756108820438385, 'kl/beta': 0.03075668215751648, 'kl/avg_steps': 0.96875, 'epoch': 0.24}
|
|||
|
|
24%|███████████████████████████▍ | 164/681 [07:33<20:42, 2.40s/it]
24%|███████████████████████████▌ | 165/681 [07:36<21:05, 2.45s/it]
{'loss': 0.4735, 'grad_norm': 75.63594055175781, 'learning_rate': 4.708572792802069e-07, 'rewards/chosen': -1.7484538555145264, 'rewards/rejected': -4.094590187072754, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.3461360931396484, 'logps/chosen': -100.64947509765625, 'logps/rejected': -213.5492401123047, 'logps/ref_chosen': -42.845787048339844, 'logps/ref_rejected': -77.92173767089844, 'logits/chosen': -2.676835536956787, 'logits/rejected': -2.463170051574707, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.03020758554339409, 'epsilon_dpo/loss_margin_mean': 77.82381439208984, 'epsilon_dpo/beta_margin_mean': 2.3461360931396484, 'epsilon_dpo/beta_margin_std': 1.85222589969635, 'epsilon_dpo/beta_margin_grad_mean': -0.17471088469028473, 'epsilon_dpo/beta_margin_grad_std': 0.19172601401805878, 'kl/beta': 0.030461585149168968, 'kl/avg_steps': 0.84375, 'epoch': 0.24}
|
|||
|
|
24%|███████████████████████████▌ | 165/681 [07:36<21:05, 2.45s/it]
24%|███████████████████████████▊ | 166/681 [07:38<21:38, 2.52s/it]
{'loss': 0.4996, 'grad_norm': 83.7566909790039, 'learning_rate': 4.702530485714461e-07, 'rewards/chosen': -1.9366867542266846, 'rewards/rejected': -4.597411632537842, 'rewards/accuracies': 0.859375, 'rewards/margins': 2.6607251167297363, 'logps/chosen': -106.1424560546875, 'logps/rejected': -259.1865234375, 'logps/ref_chosen': -41.773406982421875, 'logps/ref_rejected': -105.80018615722656, 'logits/chosen': -2.4904098510742188, 'logits/rejected': -2.599632740020752, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.03000204637646675, 'epsilon_dpo/loss_margin_mean': 89.0173110961914, 'epsilon_dpo/beta_margin_mean': 2.6607251167297363, 'epsilon_dpo/beta_margin_std': 2.3676037788391113, 'epsilon_dpo/beta_margin_grad_mean': -0.1740747094154358, 'epsilon_dpo/beta_margin_grad_std': 0.21660596132278442, 'kl/beta': 0.030206717550754547, 'kl/avg_steps': 0.6875, 'epoch': 0.24}
|
|||
|
|
24%|███████████████████████████▊ | 166/681 [07:38<21:38, 2.52s/it]
25%|███████████████████████████▉ | 167/681 [07:41<21:50, 2.55s/it]
{'loss': 0.3122, 'grad_norm': 57.45083999633789, 'learning_rate': 4.6964301399001877e-07, 'rewards/chosen': -1.8347046375274658, 'rewards/rejected': -4.77065372467041, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.9359493255615234, 'logps/chosen': -93.92123413085938, 'logps/rejected': -260.33837890625, 'logps/ref_chosen': -32.289695739746094, 'logps/ref_rejected': -99.84996795654297, 'logits/chosen': -2.4179039001464844, 'logits/rejected': -2.5366764068603516, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.02974093332886696, 'epsilon_dpo/loss_margin_mean': 98.85689544677734, 'epsilon_dpo/beta_margin_mean': 2.9359490871429443, 'epsilon_dpo/beta_margin_std': 1.7715035676956177, 'epsilon_dpo/beta_margin_grad_mean': -0.12238018214702606, 'epsilon_dpo/beta_margin_grad_std': 0.16405010223388672, 'kl/beta': 0.030000463128089905, 'kl/avg_steps': 0.875, 'epoch': 0.25}
|
|||
|
|
25%|███████████████████████████▉ | 167/681 [07:41<21:50, 2.55s/it]
25%|████████████████████████████ | 168/681 [07:43<21:35, 2.53s/it]
{'loss': 0.4007, 'grad_norm': 70.27559661865234, 'learning_rate': 4.690271916109034e-07, 'rewards/chosen': -1.767836093902588, 'rewards/rejected': -4.313807964324951, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.5459718704223633, 'logps/chosen': -95.75901794433594, 'logps/rejected': -226.67669677734375, 'logps/ref_chosen': -35.88120651245117, 'logps/ref_rejected': -80.32093811035156, 'logits/chosen': -2.536531448364258, 'logits/rejected': -2.3941898345947266, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.02948295697569847, 'epsilon_dpo/loss_margin_mean': 86.47795104980469, 'epsilon_dpo/beta_margin_mean': 2.5459718704223633, 'epsilon_dpo/beta_margin_std': 1.8775861263275146, 'epsilon_dpo/beta_margin_grad_mean': -0.15910814702510834, 'epsilon_dpo/beta_margin_grad_std': 0.168760284781456, 'kl/beta': 0.029740236699581146, 'kl/avg_steps': 0.875, 'epoch': 0.25}
|
|||
|
|
25%|████████████████████████████ | 168/681 [07:43<21:35, 2.53s/it]
25%|████████████████████████████▎ | 169/681 [07:46<21:44, 2.55s/it]
{'loss': 0.5701, 'grad_norm': 84.80979919433594, 'learning_rate': 4.6840559766159235e-07, 'rewards/chosen': -1.786689281463623, 'rewards/rejected': -4.318882465362549, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.532193183898926, 'logps/chosen': -96.68545532226562, 'logps/rejected': -236.14691162109375, 'logps/ref_chosen': -35.75175476074219, 'logps/ref_rejected': -88.39851379394531, 'logits/chosen': -2.558346748352051, 'logits/rejected': -2.5368800163269043, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.029254861176013947, 'epsilon_dpo/loss_margin_mean': 86.81468963623047, 'epsilon_dpo/beta_margin_mean': 2.5321929454803467, 'epsilon_dpo/beta_margin_std': 2.3219850063323975, 'epsilon_dpo/beta_margin_grad_mean': -0.19129043817520142, 'epsilon_dpo/beta_margin_grad_std': 0.22424982488155365, 'kl/beta': 0.029482265934348106, 'kl/avg_steps': 0.78125, 'epoch': 0.25}
|
|||
|
|
25%|████████████████████████████▎ | 169/681 [07:46<21:44, 2.55s/it]
25%|████████████████████████████▍ | 170/681 [07:48<21:29, 2.52s/it]
{'loss': 0.2813, 'grad_norm': 43.760440826416016, 'learning_rate': 4.6777824852166437e-07, 'rewards/chosen': -1.5055475234985352, 'rewards/rejected': -4.438089370727539, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.932541847229004, 'logps/chosen': -83.35134887695312, 'logps/rejected': -235.6991729736328, 'logps/ref_chosen': -31.416187286376953, 'logps/ref_rejected': -82.51651000976562, 'logits/chosen': -2.3630564212799072, 'logits/rejected': -2.373150587081909, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.028982365503907204, 'epsilon_dpo/loss_margin_mean': 101.24751281738281, 'epsilon_dpo/beta_margin_mean': 2.932541847229004, 'epsilon_dpo/beta_margin_std': 1.7402071952819824, 'epsilon_dpo/beta_margin_grad_mean': -0.11600478738546371, 'epsilon_dpo/beta_margin_grad_std': 0.14576373994350433, 'kl/beta': 0.029253721237182617, 'kl/avg_steps': 0.9375, 'epoch': 0.25}
|
|||
|
|
25%|████████████████████████████▍ | 170/681 [07:48<21:29, 2.52s/it]
25%|████████████████████████████▋ | 171/681 [07:51<20:54, 2.46s/it]
{'loss': 0.3355, 'grad_norm': 72.2908706665039, 'learning_rate': 4.6714516072235273e-07, 'rewards/chosen': -1.8903237581253052, 'rewards/rejected': -4.792015552520752, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.9016919136047363, 'logps/chosen': -120.83259582519531, 'logps/rejected': -283.8696594238281, 'logps/ref_chosen': -55.1216926574707, 'logps/ref_rejected': -117.03608703613281, 'logits/chosen': -2.617936611175537, 'logits/rejected': -2.546191692352295, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.028740353882312775, 'epsilon_dpo/loss_margin_mean': 101.12266540527344, 'epsilon_dpo/beta_margin_mean': 2.9016919136047363, 'epsilon_dpo/beta_margin_std': 2.042875051498413, 'epsilon_dpo/beta_margin_grad_mean': -0.13344722986221313, 'epsilon_dpo/beta_margin_grad_std': 0.16077838838100433, 'kl/beta': 0.028982015326619148, 'kl/avg_steps': 0.84375, 'epoch': 0.25}
|
|||
|
|
25%|████████████████████████████▋ | 171/681 [07:51<20:54, 2.46s/it]
25%|████████████████████████████▊ | 172/681 [07:53<20:50, 2.46s/it]
{'loss': 0.3457, 'grad_norm': 62.24739456176758, 'learning_rate': 4.6650635094610966e-07, 'rewards/chosen': -1.7575478553771973, 'rewards/rejected': -4.711941719055176, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.9543943405151367, 'logps/chosen': -103.77349090576172, 'logps/rejected': -256.19970703125, 'logps/ref_chosen': -42.184104919433594, 'logps/ref_rejected': -90.7464599609375, 'logits/chosen': -2.4678688049316406, 'logits/rejected': -2.340376853942871, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.028499886393547058, 'epsilon_dpo/loss_margin_mean': 103.86387634277344, 'epsilon_dpo/beta_margin_mean': 2.9543943405151367, 'epsilon_dpo/beta_margin_std': 1.9407035112380981, 'epsilon_dpo/beta_margin_grad_mean': -0.12330116331577301, 'epsilon_dpo/beta_margin_grad_std': 0.18182381987571716, 'kl/beta': 0.028739525005221367, 'kl/avg_steps': 0.84375, 'epoch': 0.25}
|
|||
|
|
25%|████████████████████████████▊ | 172/681 [07:53<20:50, 2.46s/it]
25%|████████████████████████████▉ | 173/681 [07:56<20:52, 2.46s/it]
{'loss': 0.4831, 'grad_norm': 93.45008850097656, 'learning_rate': 4.6586183602616687e-07, 'rewards/chosen': -2.261880874633789, 'rewards/rejected': -4.680129051208496, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.418248414993286, 'logps/chosen': -126.22164154052734, 'logps/rejected': -249.71383666992188, 'logps/ref_chosen': -46.40528869628906, 'logps/ref_rejected': -84.17361450195312, 'logits/chosen': -2.362546443939209, 'logits/rejected': -2.276439666748047, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.028279244899749756, 'epsilon_dpo/loss_margin_mean': 85.72386932373047, 'epsilon_dpo/beta_margin_mean': 2.418248414993286, 'epsilon_dpo/beta_margin_std': 1.9150099754333496, 'epsilon_dpo/beta_margin_grad_mean': -0.1761482208967209, 'epsilon_dpo/beta_margin_grad_std': 0.1999354213476181, 'kl/beta': 0.028499064967036247, 'kl/avg_steps': 0.78125, 'epoch': 0.25}
|
|||
|
|
25%|████████████████████████████▉ | 173/681 [07:56<20:52, 2.46s/it]
26%|█████████████████████████████▏ | 174/681 [07:58<20:47, 2.46s/it]
{'loss': 0.4027, 'grad_norm': 68.29830932617188, 'learning_rate': 4.652116329460919e-07, 'rewards/chosen': -2.0658488273620605, 'rewards/rejected': -4.852049827575684, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.786201238632202, 'logps/chosen': -106.728515625, 'logps/rejected': -280.6492004394531, 'logps/ref_chosen': -33.12921142578125, 'logps/ref_rejected': -107.51828002929688, 'logits/chosen': -2.276648759841919, 'logits/rejected': -2.400620460510254, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.028042349964380264, 'epsilon_dpo/loss_margin_mean': 99.53160858154297, 'epsilon_dpo/beta_margin_mean': 2.786201238632202, 'epsilon_dpo/beta_margin_std': 2.2568979263305664, 'epsilon_dpo/beta_margin_grad_mean': -0.15671639144420624, 'epsilon_dpo/beta_margin_grad_std': 0.1755547821521759, 'kl/beta': 0.02827814221382141, 'kl/avg_steps': 0.84375, 'epoch': 0.26}
|
|||
|
|
26%|█████████████████████████████▏ | 174/681 [07:58<20:47, 2.46s/it]
26%|█████████████████████████████▎ | 175/681 [08:01<20:41, 2.45s/it]
{'loss': 0.3301, 'grad_norm': 80.84042358398438, 'learning_rate': 4.645557588393406e-07, 'rewards/chosen': -2.0765433311462402, 'rewards/rejected': -5.386348724365234, 'rewards/accuracies': 0.921875, 'rewards/margins': 3.309805393218994, 'logps/chosen': -105.72348022460938, 'logps/rejected': -290.5704345703125, 'logps/ref_chosen': -31.185977935791016, 'logps/ref_rejected': -96.78857421875, 'logits/chosen': -2.220841884613037, 'logits/rejected': -2.1725313663482666, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.027807721868157387, 'epsilon_dpo/loss_margin_mean': 119.24435424804688, 'epsilon_dpo/beta_margin_mean': 3.309805393218994, 'epsilon_dpo/beta_margin_std': 2.263869524002075, 'epsilon_dpo/beta_margin_grad_mean': -0.12103898078203201, 'epsilon_dpo/beta_margin_grad_std': 0.18520337343215942, 'kl/beta': 0.028041541576385498, 'kl/avg_steps': 0.84375, 'epoch': 0.26}
|
|||
|
|
26%|█████████████████████████████▎ | 175/681 [08:01<20:41, 2.45s/it]
26%|█████████████████████████████▍ | 176/681 [08:03<20:26, 2.43s/it]
{'loss': 0.3514, 'grad_norm': 60.13013458251953, 'learning_rate': 4.638942309888058e-07, 'rewards/chosen': -2.254232168197632, 'rewards/rejected': -5.233524322509766, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.979292392730713, 'logps/chosen': -116.0774917602539, 'logps/rejected': -293.26025390625, 'logps/ref_chosen': -34.4107666015625, 'logps/ref_rejected': -103.36322021484375, 'logits/chosen': -2.2297158241271973, 'logits/rejected': -2.2983927726745605, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.02758374810218811, 'epsilon_dpo/loss_margin_mean': 108.23030853271484, 'epsilon_dpo/beta_margin_mean': 2.979292154312134, 'epsilon_dpo/beta_margin_std': 2.1362128257751465, 'epsilon_dpo/beta_margin_grad_mean': -0.1377795934677124, 'epsilon_dpo/beta_margin_grad_std': 0.17307686805725098, 'kl/beta': 0.027806920930743217, 'kl/avg_steps': 0.8125, 'epoch': 0.26}
|
|||
|
|
26%|█████████████████████████████▍ | 176/681 [08:03<20:26, 2.43s/it]
26%|█████████████████████████████▋ | 177/681 [08:06<20:47, 2.48s/it]
{'loss': 0.4875, 'grad_norm': 88.59068298339844, 'learning_rate': 4.6322706682636137e-07, 'rewards/chosen': -2.2037370204925537, 'rewards/rejected': -5.093808650970459, 'rewards/accuracies': 0.828125, 'rewards/margins': 2.8900716304779053, 'logps/chosen': -122.57586669921875, 'logps/rejected': -287.5384826660156, 'logps/ref_chosen': -42.30748748779297, 'logps/ref_rejected': -101.40332794189453, 'logits/chosen': -2.3562376499176025, 'logits/rejected': -2.3722541332244873, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.027404537424445152, 'epsilon_dpo/loss_margin_mean': 105.86678314208984, 'epsilon_dpo/beta_margin_mean': 2.8900716304779053, 'epsilon_dpo/beta_margin_std': 2.369600296020508, 'epsilon_dpo/beta_margin_grad_mean': -0.17347557842731476, 'epsilon_dpo/beta_margin_grad_std': 0.22418558597564697, 'kl/beta': 0.027582809329032898, 'kl/avg_steps': 0.65625, 'epoch': 0.26}
|
|||
|
|
26%|█████████████████████████████▋ | 177/681 [08:06<20:47, 2.48s/it]
26%|█████████████████████████████▊ | 178/681 [08:08<21:00, 2.51s/it]
{'loss': 0.2914, 'grad_norm': 60.96648025512695, 'learning_rate': 4.6255428393240354e-07, 'rewards/chosen': -2.04891300201416, 'rewards/rejected': -5.5481085777282715, 'rewards/accuracies': 0.953125, 'rewards/margins': 3.4991955757141113, 'logps/chosen': -107.5852279663086, 'logps/rejected': -316.2816467285156, 'logps/ref_chosen': -32.16440963745117, 'logps/ref_rejected': -111.8971939086914, 'logits/chosen': -2.2090775966644287, 'logits/rejected': -2.2421045303344727, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.027148790657520294, 'epsilon_dpo/loss_margin_mean': 128.963623046875, 'epsilon_dpo/beta_margin_mean': 3.4991955757141113, 'epsilon_dpo/beta_margin_std': 2.5016074180603027, 'epsilon_dpo/beta_margin_grad_mean': -0.11924877017736435, 'epsilon_dpo/beta_margin_grad_std': 0.15244512259960175, 'kl/beta': 0.027402978390455246, 'kl/avg_steps': 0.9375, 'epoch': 0.26}
|
|||
|
|
26%|█████████████████████████████▊ | 178/681 [08:08<21:00, 2.51s/it]
26%|█████████████████████████████▉ | 179/681 [08:11<21:12, 2.53s/it]
{'loss': 0.4834, 'grad_norm': 76.76069641113281, 'learning_rate': 4.6187590003538724e-07, 'rewards/chosen': -2.1310267448425293, 'rewards/rejected': -5.084205627441406, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.953178882598877, 'logps/chosen': -120.60657501220703, 'logps/rejected': -285.45098876953125, 'logps/ref_chosen': -41.662925720214844, 'logps/ref_rejected': -96.57734680175781, 'logits/chosen': -2.2808642387390137, 'logits/rejected': -2.315495252609253, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.026943257078528404, 'epsilon_dpo/loss_margin_mean': 109.93001556396484, 'epsilon_dpo/beta_margin_mean': 2.953178882598877, 'epsilon_dpo/beta_margin_std': 2.3476455211639404, 'epsilon_dpo/beta_margin_grad_mean': -0.16198381781578064, 'epsilon_dpo/beta_margin_grad_std': 0.22395355999469757, 'kl/beta': 0.02714846096932888, 'kl/avg_steps': 0.765625, 'epoch': 0.26}
|
|||
|
|
26%|█████████████████████████████▉ | 179/681 [08:11<21:12, 2.53s/it]
26%|██████████████████████████████▏ | 180/681 [08:13<20:57, 2.51s/it]
{'loss': 0.4102, 'grad_norm': 90.68710327148438, 'learning_rate': 4.611919330113591e-07, 'rewards/chosen': -2.2053043842315674, 'rewards/rejected': -5.2506513595581055, 'rewards/accuracies': 0.921875, 'rewards/margins': 3.045346975326538, 'logps/chosen': -128.1096649169922, 'logps/rejected': -303.2843017578125, 'logps/ref_chosen': -45.704586029052734, 'logps/ref_rejected': -106.64442443847656, 'logits/chosen': -2.333063840866089, 'logits/rejected': -2.4003429412841797, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.02671753242611885, 'epsilon_dpo/loss_margin_mean': 114.23480987548828, 'epsilon_dpo/beta_margin_mean': 3.045346975326538, 'epsilon_dpo/beta_margin_std': 2.1593165397644043, 'epsilon_dpo/beta_margin_grad_mean': -0.13764944672584534, 'epsilon_dpo/beta_margin_grad_std': 0.20955896377563477, 'kl/beta': 0.026942186057567596, 'kl/avg_steps': 0.84375, 'epoch': 0.26}
|
|||
|
|
26%|██████████████████████████████▏ | 180/681 [08:13<20:57, 2.51s/it]
27%|██████████████████████████████▎ | 181/681 [08:16<21:11, 2.54s/it]
{'loss': 0.5481, 'grad_norm': 90.5920181274414, 'learning_rate': 4.605024008834863e-07, 'rewards/chosen': -2.093709945678711, 'rewards/rejected': -4.527114391326904, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.4334044456481934, 'logps/chosen': -121.45870971679688, 'logps/rejected': -237.80453491210938, 'logps/ref_chosen': -42.68443298339844, 'logps/ref_rejected': -66.97550201416016, 'logits/chosen': -2.3339180946350098, 'logits/rejected': -2.1919121742248535, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.026519039645791054, 'epsilon_dpo/loss_margin_mean': 92.05476379394531, 'epsilon_dpo/beta_margin_mean': 2.4334042072296143, 'epsilon_dpo/beta_margin_std': 2.0837597846984863, 'epsilon_dpo/beta_margin_grad_mean': -0.17692674696445465, 'epsilon_dpo/beta_margin_grad_std': 0.21322111785411835, 'kl/beta': 0.02671676315367222, 'kl/avg_steps': 0.75, 'epoch': 0.27}
|
|||
|
|
27%|██████████████████████████████▎ | 181/681 [08:16<21:11, 2.54s/it]
27%|██████████████████████████████▍ | 182/681 [08:18<21:01, 2.53s/it]
{'loss': 0.402, 'grad_norm': 73.83556365966797, 'learning_rate': 4.598073218215817e-07, 'rewards/chosen': -1.9256913661956787, 'rewards/rejected': -5.0302886962890625, 'rewards/accuracies': 0.90625, 'rewards/margins': 3.104597568511963, 'logps/chosen': -95.87370300292969, 'logps/rejected': -287.557373046875, 'logps/ref_chosen': -22.746728897094727, 'logps/ref_rejected': -96.14221954345703, 'logits/chosen': -2.208120822906494, 'logits/rejected': -2.3168540000915527, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.026296764612197876, 'epsilon_dpo/loss_margin_mean': 118.28819274902344, 'epsilon_dpo/beta_margin_mean': 3.104597330093384, 'epsilon_dpo/beta_margin_std': 2.1538186073303223, 'epsilon_dpo/beta_margin_grad_mean': -0.14079764485359192, 'epsilon_dpo/beta_margin_grad_std': 0.20722897350788116, 'kl/beta': 0.026517879217863083, 'kl/avg_steps': 0.84375, 'epoch': 0.27}
|
|||
|
|
27%|██████████████████████████████▍ | 182/681 [08:18<21:01, 2.53s/it]
27%|██████████████████████████████▋ | 183/681 [08:20<20:16, 2.44s/it]
{'loss': 0.305, 'grad_norm': 49.043949127197266, 'learning_rate': 4.5910671414162484e-07, 'rewards/chosen': -1.921753168106079, 'rewards/rejected': -4.735166549682617, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.813413619995117, 'logps/chosen': -111.11225891113281, 'logps/rejected': -263.7894287109375, 'logps/ref_chosen': -37.3929557800293, 'logps/ref_rejected': -82.01314544677734, 'logits/chosen': -2.3619985580444336, 'logits/rejected': -2.227496862411499, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.02606852352619171, 'epsilon_dpo/loss_margin_mean': 108.05697631835938, 'epsilon_dpo/beta_margin_mean': 2.813413619995117, 'epsilon_dpo/beta_margin_std': 1.8080170154571533, 'epsilon_dpo/beta_margin_grad_mean': -0.12661683559417725, 'epsilon_dpo/beta_margin_grad_std': 0.14383679628372192, 'kl/beta': 0.02629600651562214, 'kl/avg_steps': 0.875, 'epoch': 0.27}
|
|||
|
|
27%|██████████████████████████████▋ | 183/681 [08:21<20:16, 2.44s/it]
27%|██████████████████████████████▊ | 184/681 [08:23<20:06, 2.43s/it]
{'loss': 0.6265, 'grad_norm': 91.00837707519531, 'learning_rate': 4.5840059630527985e-07, 'rewards/chosen': -2.1323318481445312, 'rewards/rejected': -4.139222145080566, 'rewards/accuracies': 0.828125, 'rewards/margins': 2.006890058517456, 'logps/chosen': -125.68778991699219, 'logps/rejected': -241.31008911132812, 'logps/ref_chosen': -43.53949737548828, 'logps/ref_rejected': -81.33880615234375, 'logits/chosen': -2.4576053619384766, 'logits/rejected': -2.333487033843994, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.02590757980942726, 'epsilon_dpo/loss_margin_mean': 77.82298278808594, 'epsilon_dpo/beta_margin_mean': 2.006890058517456, 'epsilon_dpo/beta_margin_std': 1.832121729850769, 'epsilon_dpo/beta_margin_grad_mean': -0.22445106506347656, 'epsilon_dpo/beta_margin_grad_std': 0.2272690087556839, 'kl/beta': 0.026067912578582764, 'kl/avg_steps': 0.625, 'epoch': 0.27}
|
|||
|
|
27%|██████████████████████████████▊ | 184/681 [08:23<20:06, 2.43s/it]
27%|██████████████████████████████▉ | 185/681 [08:25<20:08, 2.44s/it]
{'loss': 0.4255, 'grad_norm': 67.63098907470703, 'learning_rate': 4.5768898691940836e-07, 'rewards/chosen': -1.9024994373321533, 'rewards/rejected': -4.268389701843262, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.3658905029296875, 'logps/chosen': -111.3865737915039, 'logps/rejected': -244.80673217773438, 'logps/ref_chosen': -37.372806549072266, 'logps/ref_rejected': -78.53436279296875, 'logits/chosen': -2.360046863555908, 'logits/rejected': -2.2244629859924316, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0256899856030941, 'epsilon_dpo/loss_margin_mean': 92.25860595703125, 'epsilon_dpo/beta_margin_mean': 2.3658905029296875, 'epsilon_dpo/beta_margin_std': 1.7580143213272095, 'epsilon_dpo/beta_margin_grad_mean': -0.16735970973968506, 'epsilon_dpo/beta_margin_grad_std': 0.17599676549434662, 'kl/beta': 0.025905998423695564, 'kl/avg_steps': 0.84375, 'epoch': 0.27}
|
|||
|
|
27%|██████████████████████████████▉ | 185/681 [08:25<20:08, 2.44s/it]
27%|███████████████████████████████▏ | 186/681 [08:28<20:14, 2.45s/it]
{'loss': 0.2184, 'grad_norm': 39.774940490722656, 'learning_rate': 4.5697190473557947e-07, 'rewards/chosen': -1.5010137557983398, 'rewards/rejected': -4.650777816772461, 'rewards/accuracies': 1.0, 'rewards/margins': 3.1497642993927, 'logps/chosen': -100.51115417480469, 'logps/rejected': -276.3628234863281, 'logps/ref_chosen': -41.56720733642578, 'logps/ref_rejected': -93.58636474609375, 'logits/chosen': -2.5002646446228027, 'logits/rejected': -2.3675756454467773, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.02545095421373844, 'epsilon_dpo/loss_margin_mean': 123.8324966430664, 'epsilon_dpo/beta_margin_mean': 3.1497642993927, 'epsilon_dpo/beta_margin_std': 1.6914327144622803, 'epsilon_dpo/beta_margin_grad_mean': -0.09410841017961502, 'epsilon_dpo/beta_margin_grad_std': 0.11750077456235886, 'kl/beta': 0.025689246132969856, 'kl/avg_steps': 0.9375, 'epoch': 0.27}
|
|||
|
|
27%|███████████████████████████████▏ | 186/681 [08:28<20:14, 2.45s/it]
27%|███████████████████████████████▎ | 187/681 [08:30<19:45, 2.40s/it]
{'loss': 0.5347, 'grad_norm': 85.82039642333984, 'learning_rate': 4.5624936864957555e-07, 'rewards/chosen': -1.861724615097046, 'rewards/rejected': -4.23075008392334, 'rewards/accuracies': 0.875, 'rewards/margins': 2.369025230407715, 'logps/chosen': -108.8873291015625, 'logps/rejected': -253.65586853027344, 'logps/ref_chosen': -35.3756217956543, 'logps/ref_rejected': -86.07025146484375, 'logits/chosen': -2.322283983230591, 'logits/rejected': -2.289004325866699, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.025262294337153435, 'epsilon_dpo/loss_margin_mean': 94.07389831542969, 'epsilon_dpo/beta_margin_mean': 2.369025468826294, 'epsilon_dpo/beta_margin_std': 1.9370827674865723, 'epsilon_dpo/beta_margin_grad_mean': -0.18915115296840668, 'epsilon_dpo/beta_margin_grad_std': 0.22492022812366486, 'kl/beta': 0.02545064687728882, 'kl/avg_steps': 0.75, 'epoch': 0.27}
|
|||
|
|
27%|███████████████████████████████▎ | 187/681 [08:30<19:45, 2.40s/it]
28%|███████████████████████████████▍ | 188/681 [08:33<20:11, 2.46s/it]
{'loss': 0.4002, 'grad_norm': 62.04313659667969, 'learning_rate': 4.5552139770089454e-07, 'rewards/chosen': -1.7442595958709717, 'rewards/rejected': -4.337894439697266, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.593634843826294, 'logps/chosen': -104.10310363769531, 'logps/rejected': -270.12518310546875, 'logps/ref_chosen': -34.50104522705078, 'logps/ref_rejected': -96.81013488769531, 'logits/chosen': -2.326641321182251, 'logits/rejected': -2.3683671951293945, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.025042656809091568, 'epsilon_dpo/loss_margin_mean': 103.7129898071289, 'epsilon_dpo/beta_margin_mean': 2.593634843826294, 'epsilon_dpo/beta_margin_std': 1.8406410217285156, 'epsilon_dpo/beta_margin_grad_mean': -0.15640155971050262, 'epsilon_dpo/beta_margin_grad_std': 0.18032178282737732, 'kl/beta': 0.02526118792593479, 'kl/avg_steps': 0.875, 'epoch': 0.28}
|
|||
|
|
28%|███████████████████████████████▍ | 188/681 [08:33<20:11, 2.46s/it]
28%|███████████████████████████████▋ | 189/681 [08:35<19:31, 2.38s/it]
{'loss': 0.3608, 'grad_norm': 57.21524429321289, 'learning_rate': 4.5478801107224794e-07, 'rewards/chosen': -1.5961487293243408, 'rewards/rejected': -4.273560523986816, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.6774120330810547, 'logps/chosen': -94.84893798828125, 'logps/rejected': -250.27532958984375, 'logps/ref_chosen': -30.625194549560547, 'logps/ref_rejected': -78.02359008789062, 'logits/chosen': -2.281872272491455, 'logits/rejected': -2.1657588481903076, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.024817608296871185, 'epsilon_dpo/loss_margin_mean': 108.02799224853516, 'epsilon_dpo/beta_margin_mean': 2.6774120330810547, 'epsilon_dpo/beta_margin_std': 1.7680622339248657, 'epsilon_dpo/beta_margin_grad_mean': -0.13081274926662445, 'epsilon_dpo/beta_margin_grad_std': 0.16288872063159943, 'kl/beta': 0.02504207007586956, 'kl/avg_steps': 0.90625, 'epoch': 0.28}
|
|||
|
|
28%|███████████████████████████████▋ | 189/681 [08:35<19:31, 2.38s/it]
28%|███████████████████████████████▊ | 190/681 [08:37<20:00, 2.45s/it]
{'loss': 0.3711, 'grad_norm': 55.00584030151367, 'learning_rate': 4.5404922808905543e-07, 'rewards/chosen': -1.5758315324783325, 'rewards/rejected': -4.282756805419922, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.706925392150879, 'logps/chosen': -105.57707214355469, 'logps/rejected': -283.9358825683594, 'logps/ref_chosen': -41.5223388671875, 'logps/ref_rejected': -109.71112823486328, 'logits/chosen': -2.403958797454834, 'logits/rejected': -2.4575366973876953, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.024594716727733612, 'epsilon_dpo/loss_margin_mean': 110.1700210571289, 'epsilon_dpo/beta_margin_mean': 2.706925392150879, 'epsilon_dpo/beta_margin_std': 1.9622080326080322, 'epsilon_dpo/beta_margin_grad_mean': -0.14998801052570343, 'epsilon_dpo/beta_margin_grad_std': 0.1647987961769104, 'kl/beta': 0.02481716312468052, 'kl/avg_steps': 0.90625, 'epoch': 0.28}
|
|||
|
|
28%|███████████████████████████████▊ | 190/681 [08:38<20:00, 2.45s/it]
28%|███████████████████████████████▉ | 191/681 [08:40<20:18, 2.49s/it]
{'loss': 0.2901, 'grad_norm': 55.07308578491211, 'learning_rate': 4.5330506821893565e-07, 'rewards/chosen': -1.554311990737915, 'rewards/rejected': -4.606304168701172, 'rewards/accuracies': 0.9375, 'rewards/margins': 3.051992416381836, 'logps/chosen': -107.12852478027344, 'logps/rejected': -304.5582580566406, 'logps/ref_chosen': -43.46269989013672, 'logps/ref_rejected': -115.54437255859375, 'logits/chosen': -2.455369234085083, 'logits/rejected': -2.3924498558044434, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.024389203637838364, 'epsilon_dpo/loss_margin_mean': 125.34806060791016, 'epsilon_dpo/beta_margin_mean': 3.051992416381836, 'epsilon_dpo/beta_margin_std': 1.9232109785079956, 'epsilon_dpo/beta_margin_grad_mean': -0.1167060062289238, 'epsilon_dpo/beta_margin_grad_std': 0.15317706763744354, 'kl/beta': 0.024594279006123543, 'kl/avg_steps': 0.84375, 'epoch': 0.28}
|
|||
|
|
28%|███████████████████████████████▉ | 191/681 [08:40<20:18, 2.49s/it]
28%|████████████████████████████████▏ | 192/681 [08:43<20:42, 2.54s/it]
{'loss': 0.4295, 'grad_norm': 76.85231018066406, 'learning_rate': 4.5255555107119336e-07, 'rewards/chosen': -1.9286646842956543, 'rewards/rejected': -4.504500865936279, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.575836181640625, 'logps/chosen': -124.95817565917969, 'logps/rejected': -296.22998046875, 'logps/ref_chosen': -45.30467224121094, 'logps/ref_rejected': -109.86914825439453, 'logits/chosen': -2.2935123443603516, 'logits/rejected': -2.236546754837036, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.024181291460990906, 'epsilon_dpo/loss_margin_mean': 106.70730590820312, 'epsilon_dpo/beta_margin_mean': 2.575836181640625, 'epsilon_dpo/beta_margin_std': 1.9563552141189575, 'epsilon_dpo/beta_margin_grad_mean': -0.16211238503456116, 'epsilon_dpo/beta_margin_grad_std': 0.18894152343273163, 'kl/beta': 0.024388499557971954, 'kl/avg_steps': 0.859375, 'epoch': 0.28}
|
|||
|
|
28%|████████████████████████████████▏ | 192/681 [08:43<20:42, 2.54s/it]
28%|████████████████████████████████▎ | 193/681 [08:45<20:32, 2.53s/it]
{'loss': 0.5015, 'grad_norm': 68.04452514648438, 'learning_rate': 4.5180069639630236e-07, 'rewards/chosen': -1.82987380027771, 'rewards/rejected': -4.01944637298584, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.189572334289551, 'logps/chosen': -114.80885314941406, 'logps/rejected': -248.93182373046875, 'logps/ref_chosen': -38.554039001464844, 'logps/ref_rejected': -81.17373657226562, 'logits/chosen': -2.299919605255127, 'logits/rejected': -2.2096662521362305, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.02397906966507435, 'epsilon_dpo/loss_margin_mean': 91.50328063964844, 'epsilon_dpo/beta_margin_mean': 2.189572334289551, 'epsilon_dpo/beta_margin_std': 1.6879116296768188, 'epsilon_dpo/beta_margin_grad_mean': -0.1852301061153412, 'epsilon_dpo/beta_margin_grad_std': 0.20167653262615204, 'kl/beta': 0.0241806972771883, 'kl/avg_steps': 0.84375, 'epoch': 0.28}
|
|||
|
|
28%|████████████████████████████████▎ | 193/681 [08:45<20:32, 2.53s/it]
28%|████████████████████████████████▍ | 194/681 [08:48<20:13, 2.49s/it]
{'loss': 0.3712, 'grad_norm': 58.625755310058594, 'learning_rate': 4.510405240853854e-07, 'rewards/chosen': -2.005694627761841, 'rewards/rejected': -4.220791816711426, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.215096950531006, 'logps/chosen': -111.03370666503906, 'logps/rejected': -241.85629272460938, 'logps/ref_chosen': -26.635162353515625, 'logps/ref_rejected': -64.14307403564453, 'logits/chosen': -2.02514386177063, 'logits/rejected': -1.9236395359039307, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.023755960166454315, 'epsilon_dpo/loss_margin_mean': 93.31468200683594, 'epsilon_dpo/beta_margin_mean': 2.215096950531006, 'epsilon_dpo/beta_margin_std': 1.2999720573425293, 'epsilon_dpo/beta_margin_grad_mean': -0.15306001901626587, 'epsilon_dpo/beta_margin_grad_std': 0.14770975708961487, 'kl/beta': 0.023978380486369133, 'kl/avg_steps': 0.9375, 'epoch': 0.28}
|
|||
|
|
28%|████████████████████████████████▍ | 194/681 [08:48<20:13, 2.49s/it]
29%|████████████████████████████████▋ | 195/681 [08:50<20:18, 2.51s/it]
{'loss': 0.316, 'grad_norm': 68.01368713378906, 'learning_rate': 4.5027505416968985e-07, 'rewards/chosen': -1.754541277885437, 'rewards/rejected': -4.503971099853516, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.749429702758789, 'logps/chosen': -107.69198608398438, 'logps/rejected': -294.6122741699219, 'logps/ref_chosen': -33.191192626953125, 'logps/ref_rejected': -103.19746398925781, 'logits/chosen': -2.1741151809692383, 'logits/rejected': -2.245378255844116, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.023535314947366714, 'epsilon_dpo/loss_margin_mean': 116.91402435302734, 'epsilon_dpo/beta_margin_mean': 2.749429702758789, 'epsilon_dpo/beta_margin_std': 1.724953055381775, 'epsilon_dpo/beta_margin_grad_mean': -0.128713458776474, 'epsilon_dpo/beta_margin_grad_std': 0.14728419482707977, 'kl/beta': 0.023755669593811035, 'kl/avg_steps': 0.9375, 'epoch': 0.29}
|
|||
|
|
29%|████████████████████████████████▋ | 195/681 [08:50<20:18, 2.51s/it]
29%|████████████████████████████████▊ | 196/681 [08:53<20:18, 2.51s/it]
{'loss': 0.3278, 'grad_norm': 46.899410247802734, 'learning_rate': 4.495043068200599e-07, 'rewards/chosen': -1.6580393314361572, 'rewards/rejected': -4.442327499389648, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.784287929534912, 'logps/chosen': -101.93032836914062, 'logps/rejected': -272.50140380859375, 'logps/ref_chosen': -30.988731384277344, 'logps/ref_rejected': -82.06416320800781, 'logits/chosen': -2.1250100135803223, 'logits/rejected': -1.9867154359817505, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.023338787257671356, 'epsilon_dpo/loss_margin_mean': 119.4956283569336, 'epsilon_dpo/beta_margin_mean': 2.784287929534912, 'epsilon_dpo/beta_margin_std': 1.7888661623001099, 'epsilon_dpo/beta_margin_grad_mean': -0.13086631894111633, 'epsilon_dpo/beta_margin_grad_std': 0.16106443107128143, 'kl/beta': 0.023535029962658882, 'kl/avg_steps': 0.84375, 'epoch': 0.29}
|
|||
|
|
29%|████████████████████████████████▊ | 196/681 [08:53<20:18, 2.51s/it]
29%|████████████████████████████████▉ | 197/681 [08:55<20:17, 2.52s/it]
{'loss': 0.4039, 'grad_norm': 68.2065200805664, 'learning_rate': 4.4872830234640493e-07, 'rewards/chosen': -1.7775802612304688, 'rewards/rejected': -4.160987854003906, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.3834075927734375, 'logps/chosen': -115.05329895019531, 'logps/rejected': -261.45648193359375, 'logps/ref_chosen': -38.329471588134766, 'logps/ref_rejected': -81.55877685546875, 'logits/chosen': -2.200084686279297, 'logits/rejected': -2.1047005653381348, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.02313622087240219, 'epsilon_dpo/loss_margin_mean': 103.17386627197266, 'epsilon_dpo/beta_margin_mean': 2.3834075927734375, 'epsilon_dpo/beta_margin_std': 1.601683259010315, 'epsilon_dpo/beta_margin_grad_mean': -0.15594223141670227, 'epsilon_dpo/beta_margin_grad_std': 0.17379499971866608, 'kl/beta': 0.023338114842772484, 'kl/avg_steps': 0.875, 'epoch': 0.29}
|
|||
|
|
29%|████████████████████████████████▉ | 197/681 [08:55<20:17, 2.52s/it]
29%|█████████████████████████████████▏ | 198/681 [08:58<20:25, 2.54s/it]
{'loss': 0.3647, 'grad_norm': 67.33406829833984, 'learning_rate': 4.479470611971645e-07, 'rewards/chosen': -1.8085954189300537, 'rewards/rejected': -4.7570390701293945, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.9484434127807617, 'logps/chosen': -111.89299011230469, 'logps/rejected': -310.3739013671875, 'logps/ref_chosen': -33.095436096191406, 'logps/ref_rejected': -102.81793212890625, 'logits/chosen': -2.1627941131591797, 'logits/rejected': -2.1062378883361816, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0229355338960886, 'epsilon_dpo/loss_margin_mean': 128.7584228515625, 'epsilon_dpo/beta_margin_mean': 2.9484434127807617, 'epsilon_dpo/beta_margin_std': 2.0632452964782715, 'epsilon_dpo/beta_margin_grad_mean': -0.13343603909015656, 'epsilon_dpo/beta_margin_grad_std': 0.17914170026779175, 'kl/beta': 0.023135676980018616, 'kl/avg_steps': 0.875, 'epoch': 0.29}
|
|||
|
|
29%|█████████████████████████████████▏ | 198/681 [08:58<20:25, 2.54s/it]
29%|█████████████████████████████████▎ | 199/681 [09:00<20:17, 2.53s/it]
{'loss': 0.3347, 'grad_norm': 61.87571334838867, 'learning_rate': 4.471606039587695e-07, 'rewards/chosen': -1.884255290031433, 'rewards/rejected': -4.55606746673584, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.6718125343322754, 'logps/chosen': -117.20360565185547, 'logps/rejected': -291.4031066894531, 'logps/ref_chosen': -34.307518005371094, 'logps/ref_rejected': -90.8328857421875, 'logits/chosen': -2.119852066040039, 'logits/rejected': -1.9955350160598755, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.0227294210344553, 'epsilon_dpo/loss_margin_mean': 117.67414855957031, 'epsilon_dpo/beta_margin_mean': 2.6718122959136963, 'epsilon_dpo/beta_margin_std': 1.751692295074463, 'epsilon_dpo/beta_margin_grad_mean': -0.13506709039211273, 'epsilon_dpo/beta_margin_grad_std': 0.15224339067935944, 'kl/beta': 0.022934995591640472, 'kl/avg_steps': 0.90625, 'epoch': 0.29}
|
|||
|
|
29%|█████████████████████████████████▎ | 199/681 [09:00<20:17, 2.53s/it]
29%|█████████████████████████████████▍ | 200/681 [09:03<20:04, 2.50s/it]
{'loss': 0.559, 'grad_norm': 93.91866302490234, 'learning_rate': 4.4636895135509966e-07, 'rewards/chosen': -1.9615219831466675, 'rewards/rejected': -4.4725871086120605, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.5110652446746826, 'logps/chosen': -130.70755004882812, 'logps/rejected': -284.80322265625, 'logps/ref_chosen': -43.81787872314453, 'logps/ref_rejected': -86.25389099121094, 'logits/chosen': -2.0332164764404297, 'logits/rejected': -1.9690152406692505, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.022539492696523666, 'epsilon_dpo/loss_margin_mean': 111.65966033935547, 'epsilon_dpo/beta_margin_mean': 2.5110652446746826, 'epsilon_dpo/beta_margin_std': 2.381103754043579, 'epsilon_dpo/beta_margin_grad_mean': -0.1900298297405243, 'epsilon_dpo/beta_margin_grad_std': 0.20542176067829132, 'kl/beta': 0.022729014977812767, 'kl/avg_steps': 0.84375, 'epoch': 0.29}
|
|||
|
|
29%|█████████████████████████████████▍ | 200/681 [09:03<20:04, 2.50s/it][INFO|trainer.py:4307] 2026-04-18 09:40:25,242 >>
|
|||
|
|
***** Running Evaluation *****
|
|||
|
|
[INFO|trainer.py:4309] 2026-04-18 09:40:25,243 >> Num examples = 2339
|
|||
|
|
[INFO|trainer.py:4312] 2026-04-18 09:40:25,243 >> Batch size = 8
|
|||
|
|
|
|||
|
|
0%| | 0/73 [00:00<?, ?it/s][A
|
|||
|
|
3%|███▏ | 2/73 [00:00<00:18, 3.85it/s][A
|
|||
|
|
4%|████▊ | 3/73 [00:01<00:26, 2.60it/s][A
|
|||
|
|
5%|██████▍ | 4/73 [00:01<00:30, 2.26it/s][A
|
|||
|
|
7%|████████ | 5/73 [00:02<00:32, 2.09it/s][A
|
|||
|
|
8%|█████████▌ | 6/73 [00:02<00:33, 2.02it/s][A
|
|||
|
|
10%|███████████▏ | 7/73 [00:03<00:32, 2.01it/s][A
|
|||
|
|
11%|████████████▊ | 8/73 [00:03<00:31, 2.06it/s][A
|
|||
|
|
12%|██████████████▍ | 9/73 [00:04<00:32, 2.00it/s][A
|
|||
|
|
14%|███████████████▉ | 10/73 [00:04<00:32, 1.95it/s][A
|
|||
|
|
15%|█████████████████▍ | 11/73 [00:05<00:31, 1.96it/s][A
|
|||
|
|
16%|███████████████████ | 12/73 [00:05<00:31, 1.97it/s][A
|
|||
|
|
18%|████████████████████▋ | 13/73 [00:06<00:31, 1.91it/s][A
|
|||
|
|
19%|██████████████████████▏ | 14/73 [00:06<00:30, 1.91it/s][A
|
|||
|
|
21%|███████████████████████▊ | 15/73 [00:07<00:30, 1.87it/s][A
|
|||
|
|
22%|█████████████████████████▍ | 16/73 [00:07<00:30, 1.85it/s][A
|
|||
|
|
23%|███████████████████████████ | 17/73 [00:08<00:29, 1.88it/s][A
|
|||
|
|
25%|████████████████████████████▌ | 18/73 [00:09<00:29, 1.85it/s][A
|
|||
|
|
26%|██████████████████████████████▏ | 19/73 [00:09<00:29, 1.84it/s][A
|
|||
|
|
27%|███████████████████████████████▊ | 20/73 [00:10<00:29, 1.83it/s][A
|
|||
|
|
29%|█████████████████████████████████▎ | 21/73 [00:10<00:28, 1.82it/s][A
|
|||
|
|
30%|██████████████████████████████████▉ | 22/73 [00:11<00:28, 1.82it/s][A
|
|||
|
|
32%|████████████████████████████████████▌ | 23/73 [00:11<00:27, 1.84it/s][A
|
|||
|
|
33%|██████████████████████████████████████▏ | 24/73 [00:12<00:26, 1.88it/s][A
|
|||
|
|
34%|███████████████████████████████████████▋ | 25/73 [00:12<00:25, 1.87it/s][A
|
|||
|
|
36%|█████████████████████████████████████████▎ | 26/73 [00:13<00:24, 1.91it/s][A
|
|||
|
|
37%|██████████████████████████████████████████▉ | 27/73 [00:13<00:22, 2.05it/s][A
|
|||
|
|
38%|████████████████████████████████████████████▍ | 28/73 [00:14<00:21, 2.05it/s][A
|
|||
|
|
40%|██████████████████████████████████████████████ | 29/73 [00:14<00:21, 2.01it/s][A
|
|||
|
|
41%|███████████████████████████████████████████████▋ | 30/73 [00:15<00:21, 1.98it/s][A
|
|||
|
|
42%|█████████████████████████████████████████████████▎ | 31/73 [00:15<00:21, 1.92it/s][A
|
|||
|
|
44%|██████████████████████████████████████████████████▊ | 32/73 [00:16<00:21, 1.94it/s][A
|
|||
|
|
45%|████████████████████████████████████████████████████▍ | 33/73 [00:16<00:20, 1.93it/s][A
|
|||
|
|
47%|██████████████████████████████████████████████████████ | 34/73 [00:17<00:19, 1.97it/s][A
|
|||
|
|
48%|███████████████████████████████████████████████████████▌ | 35/73 [00:17<00:19, 1.91it/s][A
|
|||
|
|
49%|█████████████████████████████████████████████████████████▏ | 36/73 [00:18<00:19, 1.92it/s][A
|
|||
|
|
51%|██████████████████████████████████████████████████████████▊ | 37/73 [00:18<00:19, 1.88it/s][A
|
|||
|
|
52%|████████████████████████████████████████████████████████████▍ | 38/73 [00:19<00:18, 1.93it/s][A
|
|||
|
|
53%|█████████████████████████████████████████████████████████████▉ | 39/73 [00:19<00:17, 1.92it/s][A
|
|||
|
|
55%|███████████████████████████████████████████████████████████████▌ | 40/73 [00:20<00:17, 1.92it/s][A
|
|||
|
|
56%|█████████████████████████████████████████████████████████████████▏ | 41/73 [00:20<00:16, 1.93it/s][A
|
|||
|
|
58%|██████████████████████████████████████████████████████████████████▋ | 42/73 [00:21<00:16, 1.90it/s][A
|
|||
|
|
59%|████████████████████████████████████████████████████████████████████▎ | 43/73 [00:22<00:16, 1.87it/s][A
|
|||
|
|
60%|█████████████████████████████████████████████████████████████████████▉ | 44/73 [00:22<00:15, 1.86it/s][A
|
|||
|
|
62%|███████████████████████████████████████████████████████████████████████▌ | 45/73 [00:23<00:15, 1.82it/s][A
|
|||
|
|
63%|█████████████████████████████████████████████████████████████████████████ | 46/73 [00:23<00:14, 1.86it/s][A
|
|||
|
|
64%|██████████████████████████████████████████████████████████████████████████▋ | 47/73 [00:24<00:13, 1.88it/s][A
|
|||
|
|
66%|████████████████████████████████████████████████████████████████████████████▎ | 48/73 [00:24<00:12, 1.93it/s][A
|
|||
|
|
67%|█████████████████████████████████████████████████████████████████████████████▊ | 49/73 [00:25<00:12, 1.88it/s][A
|
|||
|
|
68%|███████████████████████████████████████████████████████████████████████████████▍ | 50/73 [00:25<00:12, 1.86it/s][A
|
|||
|
|
70%|█████████████████████████████████████████████████████████████████████████████████ | 51/73 [00:26<00:11, 1.83it/s][A
|
|||
|
|
71%|██████████████████████████████████████████████████████████████████████████████████▋ | 52/73 [00:26<00:11, 1.82it/s][A
|
|||
|
|
73%|████████████████████████████████████████████████████████████████████████████████████▏ | 53/73 [00:27<00:10, 1.94it/s][A
|
|||
|
|
74%|█████████████████████████████████████████████████████████████████████████████████████▊ | 54/73 [00:27<00:09, 1.96it/s][A
|
|||
|
|
75%|███████████████████████████████████████████████████████████████████████████████████████▍ | 55/73 [00:28<00:08, 2.01it/s][A
|
|||
|
|
77%|████████████████████████████████████████████████████████████████████████████████████████▉ | 56/73 [00:28<00:08, 1.96it/s][A
|
|||
|
|
78%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 57/73 [00:29<00:08, 1.91it/s][A
|
|||
|
|
79%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 58/73 [00:29<00:07, 1.95it/s][A
|
|||
|
|
81%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 59/73 [00:30<00:07, 1.88it/s][A
|
|||
|
|
82%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 60/73 [00:31<00:06, 1.89it/s][A
|
|||
|
|
84%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 61/73 [00:31<00:06, 1.87it/s][A
|
|||
|
|
85%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 62/73 [00:32<00:05, 1.85it/s][A
|
|||
|
|
86%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 63/73 [00:32<00:05, 1.91it/s][A
|
|||
|
|
88%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 64/73 [00:33<00:04, 1.93it/s][A
|
|||
|
|
89%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 65/73 [00:33<00:04, 1.88it/s][A
|
|||
|
|
90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 66/73 [00:34<00:03, 1.87it/s][A
|
|||
|
|
92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 67/73 [00:34<00:03, 1.85it/s][A
|
|||
|
|
93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 68/73 [00:35<00:02, 1.87it/s][A
|
|||
|
|
95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 69/73 [00:35<00:02, 1.98it/s][A
|
|||
|
|
96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 70/73 [00:36<00:01, 1.91it/s][A
|
|||
|
|
97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 71/73 [00:36<00:01, 1.90it/s][A
|
|||
|
|
99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 72/73 [00:37<00:00, 1.88it/s][A
|
|||
|
|
100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 73/73 [00:37<00:00, 1.99it/s][A
|
|||
|
|
[A{'eval_loss': 0.4013729393482208, 'eval_runtime': 38.3575, 'eval_samples_per_second': 60.979, 'eval_steps_per_second': 1.929, 'eval_epsilon_dpo/beta': 0.022413820028305054, 'eval_epsilon_dpo/loss_margin_mean': 78.65644836425781, 'eval_epsilon_dpo/beta_margin_mean': 1.7521748542785645, 'eval_epsilon_dpo/beta_margin_std': 1.9999192953109741, 'eval_epsilon_dpo/beta_margin_grad_mean': -0.26273271441459656, 'eval_epsilon_dpo/beta_margin_grad_std': 0.2419932633638382, 'eval_rewards/chosen': -2.4182353019714355, 'eval_rewards/rejected': -4.170409679412842, 'eval_rewards/accuracies': 0.798373281955719, 'eval_rewards/margins': 1.7521746158599854, 'eval_logps/chosen': -175.93331909179688, 'eval_logps/rejected': -278.3790588378906, 'eval_logps/ref_chosen': -68.29110717773438, 'eval_logps/ref_rejected': -92.08038330078125, 'eval_logits/chosen': -2.2068309783935547, 'eval_logits/rejected': -2.105496644973755, 'eval_kl/p_epsilon_steps': 0.7821061611175537, 'eval_kl/n_epsilon_steps': 0.21746575832366943, 'epoch': 0.29}
|
|||
|
|
29%|█████████████████████████████████▍ | 200/681 [09:41<20:04, 2.50s/it]
|
|||
|
|
100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 73/73 [00:37<00:00, 1.99it/s][A
|
|||
|
|
[A[INFO|trainer.py:3984] 2026-04-18 09:41:20,197 >> Saving model checkpoint to /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-200
|
|||
|
|
[INFO|configuration_utils.py:419] 2026-04-18 09:41:20,296 >> Configuration saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-200/config.json
|
|||
|
|
[INFO|configuration_utils.py:911] 2026-04-18 09:41:20,394 >> Configuration saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-200/generation_config.json
|
|||
|
|
[INFO|modeling_utils.py:3580] 2026-04-18 09:42:14,171 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 6 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-200/model.safetensors.index.json.
|
|||
|
|
[INFO|tokenization_utils_base.py:2510] 2026-04-18 09:42:14,185 >> tokenizer config file saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-200/tokenizer_config.json
|
|||
|
|
[INFO|tokenization_utils_base.py:2519] 2026-04-18 09:42:14,190 >> Special tokens file saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-200/special_tokens_map.json
|
|||
|
|
30%|████████████████████████████████▊ | 201/681 [14:16<12:45:53, 95.74s/it]
{'loss': 0.4443, 'grad_norm': 77.21038055419922, 'learning_rate': 4.455721242469372e-07, 'rewards/chosen': -1.7362768650054932, 'rewards/rejected': -4.34990119934082, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.613624095916748, 'logps/chosen': -133.12782287597656, 'logps/rejected': -316.4865417480469, 'logps/ref_chosen': -55.676116943359375, 'logps/ref_rejected': -121.86392974853516, 'logits/chosen': -2.2331690788269043, 'logits/rejected': -2.180114269256592, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.02235795184969902, 'epsilon_dpo/loss_margin_mean': 117.17089080810547, 'epsilon_dpo/beta_margin_mean': 2.613624095916748, 'epsilon_dpo/beta_margin_std': 1.999254584312439, 'epsilon_dpo/beta_margin_grad_mean': -0.1633475422859192, 'epsilon_dpo/beta_margin_grad_std': 0.2009790688753128, 'kl/beta': 0.022538842633366585, 'kl/avg_steps': 0.8125, 'epoch': 0.3}
|
|||
|
|
30%|████████████████████████████████▊ | 201/681 [14:16<12:45:53, 95.74s/it]
30%|█████████████████████████████████▏ | 202/681 [14:19<9:01:16, 67.80s/it]
{'loss': 0.4009, 'grad_norm': 63.114845275878906, 'learning_rate': 4.4477014363141755e-07, 'rewards/chosen': -1.8372435569763184, 'rewards/rejected': -4.240664482116699, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.403420925140381, 'logps/chosen': -113.52767181396484, 'logps/rejected': -284.8798828125, 'logps/ref_chosen': -30.73172378540039, 'logps/ref_rejected': -93.48927307128906, 'logits/chosen': -1.9032566547393799, 'logits/rejected': -2.0256400108337402, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.022170767188072205, 'epsilon_dpo/loss_margin_mean': 108.59467315673828, 'epsilon_dpo/beta_margin_mean': 2.40342116355896, 'epsilon_dpo/beta_margin_std': 1.810045599937439, 'epsilon_dpo/beta_margin_grad_mean': -0.16092374920845032, 'epsilon_dpo/beta_margin_grad_std': 0.16366447508335114, 'kl/beta': 0.022357190027832985, 'kl/avg_steps': 0.84375, 'epoch': 0.3}
|
|||
|
|
30%|█████████████████████████████████▏ | 202/681 [14:19<9:01:16, 67.80s/it]
30%|█████████████████████████████████▍ | 203/681 [14:21<6:24:19, 48.24s/it]
{'loss': 0.3417, 'grad_norm': 45.17288589477539, 'learning_rate': 4.439630306414758e-07, 'rewards/chosen': -1.5144764184951782, 'rewards/rejected': -4.293052673339844, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.778576612472534, 'logps/chosen': -107.68612670898438, 'logps/rejected': -287.6023864746094, 'logps/ref_chosen': -38.8436393737793, 'logps/ref_rejected': -92.13667297363281, 'logits/chosen': -2.110769748687744, 'logits/rejected': -2.13863468170166, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.021971412003040314, 'epsilon_dpo/loss_margin_mean': 126.62322998046875, 'epsilon_dpo/beta_margin_mean': 2.778576612472534, 'epsilon_dpo/beta_margin_std': 1.7826684713363647, 'epsilon_dpo/beta_margin_grad_mean': -0.12936589121818542, 'epsilon_dpo/beta_margin_grad_std': 0.16617560386657715, 'kl/beta': 0.022170130163431168, 'kl/avg_steps': 0.90625, 'epoch': 0.3}
|
|||
|
|
30%|█████████████████████████████████▍ | 203/681 [14:21<6:24:19, 48.24s/it]
30%|█████████████████████████████████▌ | 204/681 [14:24<4:35:10, 34.61s/it]
{'loss': 0.3937, 'grad_norm': 67.1108169555664, 'learning_rate': 4.431508065452897e-07, 'rewards/chosen': -1.7504637241363525, 'rewards/rejected': -4.320202350616455, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.5697386264801025, 'logps/chosen': -135.9166259765625, 'logps/rejected': -292.0838623046875, 'logps/ref_chosen': -55.713932037353516, 'logps/ref_rejected': -93.70796203613281, 'logits/chosen': -2.281491756439209, 'logits/rejected': -2.114370822906494, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.02179468236863613, 'epsilon_dpo/loss_margin_mean': 118.17318725585938, 'epsilon_dpo/beta_margin_mean': 2.5697386264801025, 'epsilon_dpo/beta_margin_std': 1.7625229358673096, 'epsilon_dpo/beta_margin_grad_mean': -0.14814503490924835, 'epsilon_dpo/beta_margin_grad_std': 0.18509677052497864, 'kl/beta': 0.021971017122268677, 'kl/avg_steps': 0.8125, 'epoch': 0.3}
|
|||
|
|
30%|█████████████████████████████████▌ | 204/681 [14:24<4:35:10, 34.61s/it]
30%|█████████████████████████████████▋ | 205/681 [14:27<3:18:23, 25.01s/it]
{'loss': 0.3485, 'grad_norm': 67.83074951171875, 'learning_rate': 4.4233349274571974e-07, 'rewards/chosen': -1.872864007949829, 'rewards/rejected': -4.799254417419434, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.9263906478881836, 'logps/chosen': -121.29883575439453, 'logps/rejected': -314.7345886230469, 'logps/ref_chosen': -34.816200256347656, 'logps/ref_rejected': -92.58261108398438, 'logits/chosen': -2.015986680984497, 'logits/rejected': -1.8826128244400024, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.021612215787172318, 'epsilon_dpo/loss_margin_mean': 135.66934204101562, 'epsilon_dpo/beta_margin_mean': 2.9263906478881836, 'epsilon_dpo/beta_margin_std': 1.8439240455627441, 'epsilon_dpo/beta_margin_grad_mean': -0.12213469296693802, 'epsilon_dpo/beta_margin_grad_std': 0.18215671181678772, 'kl/beta': 0.021793941035866737, 'kl/avg_steps': 0.84375, 'epoch': 0.3}
|
|||
|
|
30%|█████████████████████████████████▋ | 205/681 [14:27<3:18:23, 25.01s/it]
30%|█████████████████████████████████▉ | 206/681 [14:29<2:24:19, 18.23s/it]
{'loss': 0.4266, 'grad_norm': 70.67012023925781, 'learning_rate': 4.415111107797445e-07, 'rewards/chosen': -1.8712209463119507, 'rewards/rejected': -4.5623250007629395, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.6911041736602783, 'logps/chosen': -117.2315673828125, 'logps/rejected': -316.34771728515625, 'logps/ref_chosen': -30.099918365478516, 'logps/ref_rejected': -103.39237976074219, 'logits/chosen': -1.9591203927993774, 'logits/rejected': -1.9519094228744507, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.021431388333439827, 'epsilon_dpo/loss_margin_mean': 125.82368469238281, 'epsilon_dpo/beta_margin_mean': 2.6911041736602783, 'epsilon_dpo/beta_margin_std': 1.9302713871002197, 'epsilon_dpo/beta_margin_grad_mean': -0.14491204917430878, 'epsilon_dpo/beta_margin_grad_std': 0.19017614424228668, 'kl/beta': 0.021611593663692474, 'kl/avg_steps': 0.84375, 'epoch': 0.3}
|
|||
|
|
30%|█████████████████████████████████▉ | 206/681 [14:29<2:24:19, 18.23s/it]
30%|██████████████████████████████████ | 207/681 [14:32<1:46:56, 13.54s/it]
{'loss': 0.3788, 'grad_norm': 59.43008041381836, 'learning_rate': 4.4068368231789365e-07, 'rewards/chosen': -1.4902305603027344, 'rewards/rejected': -4.307093143463135, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.8168625831604004, 'logps/chosen': -101.26658630371094, 'logps/rejected': -292.57171630859375, 'logps/ref_chosen': -31.34187889099121, 'logps/ref_rejected': -89.86247253417969, 'logits/chosen': -2.092212200164795, 'logits/rejected': -1.968192458152771, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.02126547135412693, 'epsilon_dpo/loss_margin_mean': 132.78456115722656, 'epsilon_dpo/beta_margin_mean': 2.8168625831604004, 'epsilon_dpo/beta_margin_std': 1.992667317390442, 'epsilon_dpo/beta_margin_grad_mean': -0.1460837423801422, 'epsilon_dpo/beta_margin_grad_std': 0.1840200126171112, 'kl/beta': 0.02143077179789543, 'kl/avg_steps': 0.78125, 'epoch': 0.3}
|
|||
|
|
30%|██████████████████████████████████ | 207/681 [14:32<1:46:56, 13.54s/it]
31%|██████████████████████████████████▏ | 208/681 [14:34<1:20:45, 10.24s/it]
{'loss': 0.2919, 'grad_norm': 52.98038101196289, 'learning_rate': 4.398512291636768e-07, 'rewards/chosen': -1.7668159008026123, 'rewards/rejected': -4.634600639343262, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.8677845001220703, 'logps/chosen': -119.51541137695312, 'logps/rejected': -320.81878662109375, 'logps/ref_chosen': -35.819129943847656, 'logps/ref_rejected': -100.89794921875, 'logits/chosen': -2.082803726196289, 'logits/rejected': -2.1149425506591797, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.02108733169734478, 'epsilon_dpo/loss_margin_mean': 136.2245635986328, 'epsilon_dpo/beta_margin_mean': 2.8677847385406494, 'epsilon_dpo/beta_margin_std': 1.6818532943725586, 'epsilon_dpo/beta_margin_grad_mean': -0.1172553300857544, 'epsilon_dpo/beta_margin_grad_std': 0.15206165611743927, 'kl/beta': 0.021264642477035522, 'kl/avg_steps': 0.84375, 'epoch': 0.31}
|
|||
|
|
31%|██████████████████████████████████▏ | 208/681 [14:34<1:20:45, 10.24s/it]
31%|██████████████████████████████████▎ | 209/681 [14:37<1:02:11, 7.91s/it]
{'loss': 0.4441, 'grad_norm': 68.60906219482422, 'learning_rate': 4.3901377325300857e-07, 'rewards/chosen': -1.573714017868042, 'rewards/rejected': -4.144985198974609, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.5712709426879883, 'logps/chosen': -114.03466796875, 'logps/rejected': -285.0487365722656, 'logps/ref_chosen': -38.91720199584961, 'logps/ref_rejected': -86.70390319824219, 'logits/chosen': -2.075434446334839, 'logits/rejected': -1.8750395774841309, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.020917484536767006, 'epsilon_dpo/loss_margin_mean': 123.22737884521484, 'epsilon_dpo/beta_margin_mean': 2.5712711811065674, 'epsilon_dpo/beta_margin_std': 1.814591646194458, 'epsilon_dpo/beta_margin_grad_mean': -0.1551363617181778, 'epsilon_dpo/beta_margin_grad_std': 0.20075276494026184, 'kl/beta': 0.02108672261238098, 'kl/avg_steps': 0.8125, 'epoch': 0.31}
|
|||
|
|
31%|██████████████████████████████████▎ | 209/681 [14:37<1:02:11, 7.91s/it]
31%|███████████████████████████████████▏ | 210/681 [14:39<49:17, 6.28s/it]
{'loss': 0.32, 'grad_norm': 44.407615661621094, 'learning_rate': 4.381713366536311e-07, 'rewards/chosen': -1.6338069438934326, 'rewards/rejected': -4.250455856323242, 'rewards/accuracies': 0.984375, 'rewards/margins': 2.6166491508483887, 'logps/chosen': -108.2236099243164, 'logps/rejected': -290.24237060546875, 'logps/ref_chosen': -29.373889923095703, 'logps/ref_rejected': -85.03504943847656, 'logits/chosen': -2.012350082397461, 'logits/rejected': -1.9750981330871582, 'kl/p_epsilon_steps': 0.984375, 'kl/n_epsilon_steps': 0.015625, 'epsilon_dpo/beta': 0.020716214552521706, 'epsilon_dpo/loss_margin_mean': 126.35758972167969, 'epsilon_dpo/beta_margin_mean': 2.6166491508483887, 'epsilon_dpo/beta_margin_std': 1.6349241733551025, 'epsilon_dpo/beta_margin_grad_mean': -0.13410209119319916, 'epsilon_dpo/beta_margin_grad_std': 0.13863912224769592, 'kl/beta': 0.02091677300632, 'kl/avg_steps': 0.96875, 'epoch': 0.31}
|
|||
|
|
31%|███████████████████████████████████▏ | 210/681 [14:39<49:17, 6.28s/it]
31%|███████████████████████████████████▎ | 211/681 [14:42<40:20, 5.15s/it]
{'loss': 0.4024, 'grad_norm': 62.367835998535156, 'learning_rate': 4.373239415645323e-07, 'rewards/chosen': -1.7274291515350342, 'rewards/rejected': -4.268545150756836, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.541116237640381, 'logps/chosen': -131.88340759277344, 'logps/rejected': -301.1044921875, 'logps/ref_chosen': -47.88237380981445, 'logps/ref_rejected': -93.18321228027344, 'logits/chosen': -2.1857690811157227, 'logits/rejected': -2.038249969482422, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.02053687535226345, 'epsilon_dpo/loss_margin_mean': 123.92025756835938, 'epsilon_dpo/beta_margin_mean': 2.54111647605896, 'epsilon_dpo/beta_margin_std': 1.720736026763916, 'epsilon_dpo/beta_margin_grad_mean': -0.1510934829711914, 'epsilon_dpo/beta_margin_grad_std': 0.18279674649238586, 'kl/beta': 0.02071608603000641, 'kl/avg_steps': 0.875, 'epoch': 0.31}
|
|||
|
|
31%|███████████████████████████████████▎ | 211/681 [14:42<40:20, 5.15s/it]
31%|███████████████████████████████████▍ | 212/681 [14:44<34:16, 4.39s/it]
{'loss': 0.3267, 'grad_norm': 56.73617935180664, 'learning_rate': 4.3647161031536086e-07, 'rewards/chosen': -1.4194426536560059, 'rewards/rejected': -4.518226623535156, 'rewards/accuracies': 0.96875, 'rewards/margins': 3.0987837314605713, 'logps/chosen': -105.17568969726562, 'logps/rejected': -333.01275634765625, 'logps/ref_chosen': -35.5427360534668, 'logps/ref_rejected': -110.93476867675781, 'logits/chosen': -2.1070785522460938, 'logits/rejected': -2.0808253288269043, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.02035231702029705, 'epsilon_dpo/loss_margin_mean': 152.44503784179688, 'epsilon_dpo/beta_margin_mean': 3.0987837314605713, 'epsilon_dpo/beta_margin_std': 2.040536403656006, 'epsilon_dpo/beta_margin_grad_mean': -0.11501824855804443, 'epsilon_dpo/beta_margin_grad_std': 0.16267718374729156, 'kl/beta': 0.0205363929271698, 'kl/avg_steps': 0.90625, 'epoch': 0.31}
|
|||
|
|
31%|███████████████████████████████████▍ | 212/681 [14:44<34:16, 4.39s/it]
31%|███████████████████████████████████▋ | 213/681 [14:47<30:24, 3.90s/it]
{'loss': 0.4116, 'grad_norm': 60.45912551879883, 'learning_rate': 4.3561436536583774e-07, 'rewards/chosen': -1.522735595703125, 'rewards/rejected': -4.057496070861816, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.5347602367401123, 'logps/chosen': -121.68977355957031, 'logps/rejected': -299.35791015625, 'logps/ref_chosen': -46.382476806640625, 'logps/ref_rejected': -98.22808074951172, 'logits/chosen': -2.2190451622009277, 'logits/rejected': -2.022303581237793, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.02018861286342144, 'epsilon_dpo/loss_margin_mean': 125.82254028320312, 'epsilon_dpo/beta_margin_mean': 2.5347602367401123, 'epsilon_dpo/beta_margin_std': 1.81096613407135, 'epsilon_dpo/beta_margin_grad_mean': -0.1576324999332428, 'epsilon_dpo/beta_margin_grad_std': 0.1839437037706375, 'kl/beta': 0.02035195380449295, 'kl/avg_steps': 0.8125, 'epoch': 0.31}
|
|||
|
|
31%|███████████████████████████████████▋ | 213/681 [14:47<30:24, 3.90s/it]
31%|███████████████████████████████████▊ | 214/681 [14:49<26:53, 3.46s/it]
{'loss': 0.5721, 'grad_norm': 80.94749450683594, 'learning_rate': 4.3475222930516473e-07, 'rewards/chosen': -1.683471441268921, 'rewards/rejected': -3.938481330871582, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.255009889602661, 'logps/chosen': -117.653564453125, 'logps/rejected': -280.5225830078125, 'logps/ref_chosen': -33.69921112060547, 'logps/ref_rejected': -83.6459732055664, 'logits/chosen': -1.925289273262024, 'logits/rejected': -1.9057788848876953, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.02001328393816948, 'epsilon_dpo/loss_margin_mean': 112.92225646972656, 'epsilon_dpo/beta_margin_mean': 2.255009889602661, 'epsilon_dpo/beta_margin_std': 1.9002856016159058, 'epsilon_dpo/beta_margin_grad_mean': -0.18691565096378326, 'epsilon_dpo/beta_margin_grad_std': 0.2173331379890442, 'kl/beta': 0.020187927410006523, 'kl/avg_steps': 0.875, 'epoch': 0.31}
|
|||
|
|
31%|███████████████████████████████████▊ | 214/681 [14:50<26:53, 3.46s/it]
32%|███████████████████████████████████▉ | 215/681 [14:52<25:31, 3.29s/it]
{'loss': 0.2971, 'grad_norm': 46.67967987060547, 'learning_rate': 4.3388522485142885e-07, 'rewards/chosen': -1.5396337509155273, 'rewards/rejected': -4.214178562164307, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.6745448112487793, 'logps/chosen': -107.9367446899414, 'logps/rejected': -312.21636962890625, 'logps/ref_chosen': -30.35393714904785, 'logps/ref_rejected': -99.64697265625, 'logits/chosen': -2.086688756942749, 'logits/rejected': -2.0053024291992188, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.019833432510495186, 'epsilon_dpo/loss_margin_mean': 134.98655700683594, 'epsilon_dpo/beta_margin_mean': 2.6745448112487793, 'epsilon_dpo/beta_margin_std': 1.592254638671875, 'epsilon_dpo/beta_margin_grad_mean': -0.12425589561462402, 'epsilon_dpo/beta_margin_grad_std': 0.13742640614509583, 'kl/beta': 0.020012814551591873, 'kl/avg_steps': 0.90625, 'epoch': 0.32}
|
|||
|
|
32%|███████████████████████████████████▉ | 215/681 [14:52<25:31, 3.29s/it]
32%|████████████████████████████████████▏ | 216/681 [14:55<23:48, 3.07s/it]
{'loss': 0.4448, 'grad_norm': 60.99223709106445, 'learning_rate': 4.330133748510036e-07, 'rewards/chosen': -1.4557723999023438, 'rewards/rejected': -3.9574337005615234, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.5016613006591797, 'logps/chosen': -102.5655746459961, 'logps/rejected': -284.519287109375, 'logps/ref_chosen': -28.687610626220703, 'logps/ref_rejected': -83.20097351074219, 'logits/chosen': -2.0426881313323975, 'logits/rejected': -1.9267804622650146, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.019673900678753853, 'epsilon_dpo/loss_margin_mean': 127.44037628173828, 'epsilon_dpo/beta_margin_mean': 2.5016613006591797, 'epsilon_dpo/beta_margin_std': 1.8782446384429932, 'epsilon_dpo/beta_margin_grad_mean': -0.1697354018688202, 'epsilon_dpo/beta_margin_grad_std': 0.19105024635791779, 'kl/beta': 0.019833076745271683, 'kl/avg_steps': 0.8125, 'epoch': 0.32}
|
|||
|
|
32%|████████████████████████████████████▏ | 216/681 [14:55<23:48, 3.07s/it]
32%|████████████████████████████████████▎ | 217/681 [14:58<22:40, 2.93s/it]
{'loss': 0.3023, 'grad_norm': 50.90324401855469, 'learning_rate': 4.3213670227794757e-07, 'rewards/chosen': -1.3119699954986572, 'rewards/rejected': -4.179328441619873, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.867358684539795, 'logps/chosen': -98.74182891845703, 'logps/rejected': -321.77667236328125, 'logps/ref_chosen': -31.528701782226562, 'logps/ref_rejected': -107.33251190185547, 'logits/chosen': -2.1286168098449707, 'logits/rejected': -2.087315559387207, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.019503042101860046, 'epsilon_dpo/loss_margin_mean': 147.23101806640625, 'epsilon_dpo/beta_margin_mean': 2.867358446121216, 'epsilon_dpo/beta_margin_std': 1.7631230354309082, 'epsilon_dpo/beta_margin_grad_mean': -0.12006211280822754, 'epsilon_dpo/beta_margin_grad_std': 0.1564583033323288, 'kl/beta': 0.01967323198914528, 'kl/avg_steps': 0.875, 'epoch': 0.32}
|
|||
|
|
32%|████████████████████████████████████▎ | 217/681 [14:58<22:40, 2.93s/it]
32%|████████████████████████████████████▍ | 218/681 [15:00<21:12, 2.75s/it]
{'loss': 0.3962, 'grad_norm': 58.98360824584961, 'learning_rate': 4.3125523023339815e-07, 'rewards/chosen': -1.39695405960083, 'rewards/rejected': -3.623495578765869, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.226541519165039, 'logps/chosen': -108.99090576171875, 'logps/rejected': -280.693603515625, 'logps/ref_chosen': -36.7948112487793, 'logps/ref_rejected': -93.1485366821289, 'logits/chosen': -2.162309169769287, 'logits/rejected': -2.1342501640319824, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.01932777464389801, 'epsilon_dpo/loss_margin_mean': 115.3489990234375, 'epsilon_dpo/beta_margin_mean': 2.22654128074646, 'epsilon_dpo/beta_margin_std': 1.3763279914855957, 'epsilon_dpo/beta_margin_grad_mean': -0.15312562882900238, 'epsilon_dpo/beta_margin_grad_std': 0.161267951130867, 'kl/beta': 0.019502583891153336, 'kl/avg_steps': 0.90625, 'epoch': 0.32}
|
|||
|
|
32%|████████████████████████████████████▍ | 218/681 [15:00<21:12, 2.75s/it]
32%|████████████████████████████████████▋ | 219/681 [15:02<20:45, 2.70s/it]
{'loss': 0.4669, 'grad_norm': 61.87991714477539, 'learning_rate': 4.303689819449636e-07, 'rewards/chosen': -1.3612611293792725, 'rewards/rejected': -3.615694046020508, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.2544326782226562, 'logps/chosen': -113.684814453125, 'logps/rejected': -280.8560791015625, 'logps/ref_chosen': -42.875755310058594, 'logps/ref_rejected': -92.20575714111328, 'logits/chosen': -2.2709426879882812, 'logits/rejected': -2.1236162185668945, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.019178351387381554, 'epsilon_dpo/loss_margin_mean': 117.84127807617188, 'epsilon_dpo/beta_margin_mean': 2.2544329166412354, 'epsilon_dpo/beta_margin_std': 1.7746726274490356, 'epsilon_dpo/beta_margin_grad_mean': -0.178512305021286, 'epsilon_dpo/beta_margin_grad_std': 0.184907004237175, 'kl/beta': 0.019327430054545403, 'kl/avg_steps': 0.78125, 'epoch': 0.32}
|
|||
|
|
32%|████████████████████████████████████▋ | 219/681 [15:02<20:45, 2.70s/it]
32%|████████████████████████████████████▊ | 220/681 [15:05<20:22, 2.65s/it]
{'loss': 0.4237, 'grad_norm': 57.75115966796875, 'learning_rate': 4.2947798076611047e-07, 'rewards/chosen': -1.4991801977157593, 'rewards/rejected': -3.63541841506958, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.1362380981445312, 'logps/chosen': -121.99055480957031, 'logps/rejected': -286.1322326660156, 'logps/ref_chosen': -43.218231201171875, 'logps/ref_rejected': -94.84095764160156, 'logits/chosen': -2.249175548553467, 'logits/rejected': -2.228762626647949, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.019017694517970085, 'epsilon_dpo/loss_margin_mean': 112.51897430419922, 'epsilon_dpo/beta_margin_mean': 2.1362380981445312, 'epsilon_dpo/beta_margin_std': 1.426170825958252, 'epsilon_dpo/beta_margin_grad_mean': -0.17227615416049957, 'epsilon_dpo/beta_margin_grad_std': 0.1603304147720337, 'kl/beta': 0.019177604466676712, 'kl/avg_steps': 0.84375, 'epoch': 0.32}
|
|||
|
|
32%|████████████████████████████████████▊ | 220/681 [15:05<20:22, 2.65s/it]
32%|████████████████████████████████████▉ | 221/681 [15:08<20:05, 2.62s/it]
{'loss': 0.2732, 'grad_norm': 44.90319061279297, 'learning_rate': 4.285822501755485e-07, 'rewards/chosen': -1.2945091724395752, 'rewards/rejected': -4.154721260070801, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.8602123260498047, 'logps/chosen': -105.45207214355469, 'logps/rejected': -333.3541259765625, 'logps/ref_chosen': -36.884986877441406, 'logps/ref_rejected': -112.87872314453125, 'logits/chosen': -2.0982179641723633, 'logits/rejected': -2.1222004890441895, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.018852630630135536, 'epsilon_dpo/loss_margin_mean': 151.9083251953125, 'epsilon_dpo/beta_margin_mean': 2.8602123260498047, 'epsilon_dpo/beta_margin_std': 1.571157693862915, 'epsilon_dpo/beta_margin_grad_mean': -0.11038573086261749, 'epsilon_dpo/beta_margin_grad_std': 0.14331035315990448, 'kl/beta': 0.01901714690029621, 'kl/avg_steps': 0.875, 'epoch': 0.32}
|
|||
|
|
32%|████████████████████████████████████▉ | 221/681 [15:08<20:05, 2.62s/it]
33%|█████████████████████████████████████▏ | 222/681 [15:10<19:43, 2.58s/it]
{'loss': 0.3942, 'grad_norm': 68.18460845947266, 'learning_rate': 4.276818137766118e-07, 'rewards/chosen': -1.4119031429290771, 'rewards/rejected': -3.8390281200408936, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.4271249771118164, 'logps/chosen': -112.70558166503906, 'logps/rejected': -311.8533935546875, 'logps/ref_chosen': -37.27526092529297, 'logps/ref_rejected': -106.37206268310547, 'logits/chosen': -2.1893157958984375, 'logits/rejected': -2.2103257179260254, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.018689103424549103, 'epsilon_dpo/loss_margin_mean': 130.051025390625, 'epsilon_dpo/beta_margin_mean': 2.4271247386932373, 'epsilon_dpo/beta_margin_std': 1.7556877136230469, 'epsilon_dpo/beta_margin_grad_mean': -0.15696805715560913, 'epsilon_dpo/beta_margin_grad_std': 0.16546384990215302, 'kl/beta': 0.018852191045880318, 'kl/avg_steps': 0.875, 'epoch': 0.33}
|
|||
|
|
33%|█████████████████████████████████████▏ | 222/681 [15:10<19:43, 2.58s/it]
33%|█████████████████████████████████████▎ | 223/681 [15:12<18:47, 2.46s/it]
{'loss': 0.542, 'grad_norm': 77.07755279541016, 'learning_rate': 4.2677669529663686e-07, 'rewards/chosen': -1.581796407699585, 'rewards/rejected': -3.775017261505127, 'rewards/accuracies': 0.859375, 'rewards/margins': 2.193220615386963, 'logps/chosen': -117.65826416015625, 'logps/rejected': -290.10894775390625, 'logps/ref_chosen': -32.709083557128906, 'logps/ref_rejected': -86.52430725097656, 'logits/chosen': -2.1241304874420166, 'logits/rejected': -2.0650038719177246, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.018556196242570877, 'epsilon_dpo/loss_margin_mean': 118.63549041748047, 'epsilon_dpo/beta_margin_mean': 2.193220615386963, 'epsilon_dpo/beta_margin_std': 1.8057856559753418, 'epsilon_dpo/beta_margin_grad_mean': -0.1877458542585373, 'epsilon_dpo/beta_margin_grad_std': 0.21679145097732544, 'kl/beta': 0.018688665702939034, 'kl/avg_steps': 0.71875, 'epoch': 0.33}
|
|||
|
|
33%|█████████████████████████████████████▎ | 223/681 [15:12<18:47, 2.46s/it]
33%|█████████████████████████████████████▍ | 224/681 [15:14<18:08, 2.38s/it]
{'loss': 0.4078, 'grad_norm': 64.31571197509766, 'learning_rate': 4.2586691858633747e-07, 'rewards/chosen': -1.429018497467041, 'rewards/rejected': -3.910590648651123, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.481572389602661, 'logps/chosen': -113.04464721679688, 'logps/rejected': -294.23968505859375, 'logps/ref_chosen': -35.54627990722656, 'logps/ref_rejected': -81.60932922363281, 'logits/chosen': -2.1345582008361816, 'logits/rejected': -1.9911662340164185, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.018406376242637634, 'epsilon_dpo/loss_margin_mean': 135.1320037841797, 'epsilon_dpo/beta_margin_mean': 2.481572389602661, 'epsilon_dpo/beta_margin_std': 1.7067841291427612, 'epsilon_dpo/beta_margin_grad_mean': -0.15500150620937347, 'epsilon_dpo/beta_margin_grad_std': 0.1830359399318695, 'kl/beta': 0.018555298447608948, 'kl/avg_steps': 0.8125, 'epoch': 0.33}
|
|||
|
|
33%|█████████████████████████████████████▍ | 224/681 [15:14<18:08, 2.38s/it]
33%|█████████████████████████████████████▋ | 225/681 [15:17<18:22, 2.42s/it]
{'loss': 0.4097, 'grad_norm': 65.03907775878906, 'learning_rate': 4.249525076191759e-07, 'rewards/chosen': -1.4013426303863525, 'rewards/rejected': -4.040144920349121, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.6388025283813477, 'logps/chosen': -111.3397216796875, 'logps/rejected': -328.50433349609375, 'logps/ref_chosen': -34.65919876098633, 'logps/ref_rejected': -106.95365905761719, 'logits/chosen': -2.1871337890625, 'logits/rejected': -2.1202986240386963, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.018246525898575783, 'epsilon_dpo/loss_margin_mean': 144.87014770507812, 'epsilon_dpo/beta_margin_mean': 2.6388025283813477, 'epsilon_dpo/beta_margin_std': 1.800585150718689, 'epsilon_dpo/beta_margin_grad_mean': -0.1459973156452179, 'epsilon_dpo/beta_margin_grad_std': 0.19097650051116943, 'kl/beta': 0.018405752256512642, 'kl/avg_steps': 0.875, 'epoch': 0.33}
|
|||
|
|
33%|█████████████████████████████████████▋ | 225/681 [15:17<18:22, 2.42s/it]
33%|█████████████████████████████████████▊ | 226/681 [15:20<18:53, 2.49s/it]
{'loss': 0.4196, 'grad_norm': 53.89804458618164, 'learning_rate': 4.2403348649073167e-07, 'rewards/chosen': -1.3841276168823242, 'rewards/rejected': -3.5787124633789062, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.194584846496582, 'logps/chosen': -120.92872619628906, 'logps/rejected': -281.99481201171875, 'logps/ref_chosen': -44.4660758972168, 'logps/ref_rejected': -84.042724609375, 'logits/chosen': -2.2922072410583496, 'logits/rejected': -2.074962615966797, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.01808825507760048, 'epsilon_dpo/loss_margin_mean': 121.48944854736328, 'epsilon_dpo/beta_margin_mean': 2.194584846496582, 'epsilon_dpo/beta_margin_std': 1.5229578018188477, 'epsilon_dpo/beta_margin_grad_mean': -0.17010191082954407, 'epsilon_dpo/beta_margin_grad_std': 0.16130515933036804, 'kl/beta': 0.01824609935283661, 'kl/avg_steps': 0.875, 'epoch': 0.33}
|
|||
|
|
33%|█████████████████████████████████████▊ | 226/681 [15:20<18:53, 2.49s/it]
33%|██████████████████████████████████████ | 227/681 [15:22<18:37, 2.46s/it]
{'loss': 0.4017, 'grad_norm': 62.88763427734375, 'learning_rate': 4.2310987941806615e-07, 'rewards/chosen': -1.3941301107406616, 'rewards/rejected': -4.054275035858154, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.6601450443267822, 'logps/chosen': -111.29264831542969, 'logps/rejected': -332.619384765625, 'logps/ref_chosen': -33.7484245300293, 'logps/ref_rejected': -106.498291015625, 'logits/chosen': -2.217491865158081, 'logits/rejected': -2.0915043354034424, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.017948314547538757, 'epsilon_dpo/loss_margin_mean': 148.5768585205078, 'epsilon_dpo/beta_margin_mean': 2.6601450443267822, 'epsilon_dpo/beta_margin_std': 1.9379621744155884, 'epsilon_dpo/beta_margin_grad_mean': -0.15359929203987122, 'epsilon_dpo/beta_margin_grad_std': 0.18651175498962402, 'kl/beta': 0.018087830394506454, 'kl/avg_steps': 0.78125, 'epoch': 0.33}
|
|||
|
|
33%|██████████████████████████████████████ | 227/681 [15:22<18:37, 2.46s/it]
33%|██████████████████████████████████████▏ | 228/681 [15:25<19:01, 2.52s/it]
{'loss': 0.3771, 'grad_norm': 55.945281982421875, 'learning_rate': 4.2218171073908463e-07, 'rewards/chosen': -1.473421335220337, 'rewards/rejected': -3.7081234455108643, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.2347021102905273, 'logps/chosen': -128.66030883789062, 'logps/rejected': -304.66015625, 'logps/ref_chosen': -45.935726165771484, 'logps/ref_rejected': -96.16656494140625, 'logits/chosen': -2.300729990005493, 'logits/rejected': -2.2182745933532715, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0177923534065485, 'epsilon_dpo/loss_margin_mean': 125.76899719238281, 'epsilon_dpo/beta_margin_mean': 2.2347021102905273, 'epsilon_dpo/beta_margin_std': 1.3810467720031738, 'epsilon_dpo/beta_margin_grad_mean': -0.15493857860565186, 'epsilon_dpo/beta_margin_grad_std': 0.1513359248638153, 'kl/beta': 0.017947614192962646, 'kl/avg_steps': 0.875, 'epoch': 0.33}
|
|||
|
|
33%|██████████████████████████████████████▏ | 228/681 [15:25<19:01, 2.52s/it]
34%|██████████████████████████████████████▎ | 229/681 [15:27<18:49, 2.50s/it]
{'loss': 0.4188, 'grad_norm': 64.01463317871094, 'learning_rate': 4.212490049118951e-07, 'rewards/chosen': -1.1469919681549072, 'rewards/rejected': -3.779086112976074, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.632094383239746, 'logps/chosen': -100.0278091430664, 'logps/rejected': -304.251220703125, 'logps/ref_chosen': -35.16382598876953, 'logps/ref_rejected': -89.91634368896484, 'logits/chosen': -2.2102646827697754, 'logits/rejected': -2.084036350250244, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.01766026020050049, 'epsilon_dpo/loss_margin_mean': 149.47091674804688, 'epsilon_dpo/beta_margin_mean': 2.632094383239746, 'epsilon_dpo/beta_margin_std': 1.839905023574829, 'epsilon_dpo/beta_margin_grad_mean': -0.155641108751297, 'epsilon_dpo/beta_margin_grad_std': 0.19922208786010742, 'kl/beta': 0.017791934311389923, 'kl/avg_steps': 0.75, 'epoch': 0.34}
|
|||
|
|
34%|██████████████████████████████████████▎ | 229/681 [15:27<18:49, 2.50s/it]
34%|██████████████████████████████████████▌ | 230/681 [15:30<18:51, 2.51s/it]
{'loss': 0.4007, 'grad_norm': 57.6724853515625, 'learning_rate': 4.203117865141635e-07, 'rewards/chosen': -1.5095224380493164, 'rewards/rejected': -3.8183393478393555, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.30881667137146, 'logps/chosen': -114.36758422851562, 'logps/rejected': -308.0694274902344, 'logps/ref_chosen': -28.29522705078125, 'logps/ref_rejected': -89.92157745361328, 'logits/chosen': -1.9427410364151, 'logits/rejected': -1.9851582050323486, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.017512237653136253, 'epsilon_dpo/loss_margin_mean': 132.07550048828125, 'epsilon_dpo/beta_margin_mean': 2.30881667137146, 'epsilon_dpo/beta_margin_std': 1.4573560953140259, 'epsilon_dpo/beta_margin_grad_mean': -0.15799804031848907, 'epsilon_dpo/beta_margin_grad_std': 0.1725914478302002, 'kl/beta': 0.017659489065408707, 'kl/avg_steps': 0.84375, 'epoch': 0.34}
|
|||
|
|
34%|██████████████████████████████████████▌ | 230/681 [15:30<18:51, 2.51s/it]
34%|██████████████████████████████████████▋ | 231/681 [15:32<19:06, 2.55s/it]
{'loss': 0.3873, 'grad_norm': 58.66816711425781, 'learning_rate': 4.1937008024246625e-07, 'rewards/chosen': -1.4385372400283813, 'rewards/rejected': -3.782843828201294, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.344306468963623, 'logps/chosen': -122.01680755615234, 'logps/rejected': -298.10369873046875, 'logps/ref_chosen': -39.274810791015625, 'logps/ref_rejected': -80.1042709350586, 'logits/chosen': -2.2687087059020996, 'logits/rejected': -2.0110936164855957, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.01736571453511715, 'epsilon_dpo/loss_margin_mean': 135.25741577148438, 'epsilon_dpo/beta_margin_mean': 2.344306468963623, 'epsilon_dpo/beta_margin_std': 1.5224120616912842, 'epsilon_dpo/beta_margin_grad_mean': -0.15104635059833527, 'epsilon_dpo/beta_margin_grad_std': 0.16789193451404572, 'kl/beta': 0.01751173473894596, 'kl/avg_steps': 0.84375, 'epoch': 0.34}
|
|||
|
|
34%|██████████████████████████████████████▋ | 231/681 [15:32<19:06, 2.55s/it]
34%|██████████████████████████████████████▊ | 232/681 [15:35<18:58, 2.53s/it]
{'loss': 0.4288, 'grad_norm': 55.898075103759766, 'learning_rate': 4.1842391091163933e-07, 'rewards/chosen': -1.3723829984664917, 'rewards/rejected': -3.6786396503448486, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.3062567710876465, 'logps/chosen': -121.8990707397461, 'logps/rejected': -302.57232666015625, 'logps/ref_chosen': -42.393104553222656, 'logps/ref_rejected': -88.90144348144531, 'logits/chosen': -2.2190351486206055, 'logits/rejected': -2.2044219970703125, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0172258447855711, 'epsilon_dpo/loss_margin_mean': 134.1649169921875, 'epsilon_dpo/beta_margin_mean': 2.3062567710876465, 'epsilon_dpo/beta_margin_std': 1.650961995124817, 'epsilon_dpo/beta_margin_grad_mean': -0.16703417897224426, 'epsilon_dpo/beta_margin_grad_std': 0.17445211112499237, 'kl/beta': 0.017365215346217155, 'kl/avg_steps': 0.8125, 'epoch': 0.34}
|
|||
|
|
34%|██████████████████████████████████████▊ | 232/681 [15:35<18:58, 2.53s/it]
34%|███████████████████████████████████████ | 233/681 [15:37<19:03, 2.55s/it]
{'loss': 0.2959, 'grad_norm': 49.05929183959961, 'learning_rate': 4.174733034541245e-07, 'rewards/chosen': -1.3310226202011108, 'rewards/rejected': -4.204285621643066, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.873262882232666, 'logps/chosen': -108.28323364257812, 'logps/rejected': -361.3421630859375, 'logps/ref_chosen': -30.483036041259766, 'logps/ref_rejected': -115.03839111328125, 'logits/chosen': -2.11576509475708, 'logits/rejected': -2.1665868759155273, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.017076246440410614, 'epsilon_dpo/loss_margin_mean': 168.5035858154297, 'epsilon_dpo/beta_margin_mean': 2.873262882232666, 'epsilon_dpo/beta_margin_std': 1.6749653816223145, 'epsilon_dpo/beta_margin_grad_mean': -0.11840350925922394, 'epsilon_dpo/beta_margin_grad_std': 0.15504033863544464, 'kl/beta': 0.01722525991499424, 'kl/avg_steps': 0.875, 'epoch': 0.34}
|
|||
|
|
34%|███████████████████████████████████████ | 233/681 [15:37<19:03, 2.55s/it]
34%|███████████████████████████████████████▏ | 234/681 [15:40<19:08, 2.57s/it]
{'loss': 0.4328, 'grad_norm': 67.56781005859375, 'learning_rate': 4.165182829193126e-07, 'rewards/chosen': -1.3788411617279053, 'rewards/rejected': -4.01970911026001, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.6408681869506836, 'logps/chosen': -111.21868133544922, 'logps/rejected': -346.22235107421875, 'logps/ref_chosen': -30.016942977905273, 'logps/ref_rejected': -108.75608825683594, 'logits/chosen': -2.0071873664855957, 'logits/rejected': -2.1877129077911377, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.01694413460791111, 'epsilon_dpo/loss_margin_mean': 156.26454162597656, 'epsilon_dpo/beta_margin_mean': 2.6408679485321045, 'epsilon_dpo/beta_margin_std': 2.0294158458709717, 'epsilon_dpo/beta_margin_grad_mean': -0.15992534160614014, 'epsilon_dpo/beta_margin_grad_std': 0.19790522754192352, 'kl/beta': 0.01707584597170353, 'kl/avg_steps': 0.78125, 'epoch': 0.34}
|
|||
|
|
34%|███████████████████████████████████████▏ | 234/681 [15:40<19:08, 2.57s/it]
35%|███████████████████████████████████████▎ | 235/681 [15:42<18:54, 2.54s/it]
{'loss': 0.4894, 'grad_norm': 77.5376205444336, 'learning_rate': 4.1555887447288255e-07, 'rewards/chosen': -1.6119556427001953, 'rewards/rejected': -3.8830833435058594, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.271127700805664, 'logps/chosen': -141.7093963623047, 'logps/rejected': -327.0941162109375, 'logps/ref_chosen': -46.081146240234375, 'logps/ref_rejected': -96.02244567871094, 'logits/chosen': -2.111543655395508, 'logits/rejected': -2.081702470779419, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.01681278459727764, 'epsilon_dpo/loss_margin_mean': 135.44342041015625, 'epsilon_dpo/beta_margin_mean': 2.271127700805664, 'epsilon_dpo/beta_margin_std': 1.7841382026672363, 'epsilon_dpo/beta_margin_grad_mean': -0.175105020403862, 'epsilon_dpo/beta_margin_grad_std': 0.1960555464029312, 'kl/beta': 0.016943475231528282, 'kl/avg_steps': 0.78125, 'epoch': 0.35}
|
|||
|
|
35%|███████████████████████████████████████▎ | 235/681 [15:42<18:54, 2.54s/it]
35%|███████████████████████████████████████▌ | 236/681 [15:45<18:40, 2.52s/it]
{'loss': 0.4048, 'grad_norm': 56.97810363769531, 'learning_rate': 4.1459510339613946e-07, 'rewards/chosen': -1.3549550771713257, 'rewards/rejected': -3.6831302642822266, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.3281750679016113, 'logps/chosen': -112.40109252929688, 'logps/rejected': -329.66357421875, 'logps/ref_chosen': -31.17489242553711, 'logps/ref_rejected': -108.55508422851562, 'logits/chosen': -2.05222487449646, 'logits/rejected': -2.147634267807007, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.01666669175028801, 'epsilon_dpo/loss_margin_mean': 139.8822784423828, 'epsilon_dpo/beta_margin_mean': 2.3281750679016113, 'epsilon_dpo/beta_margin_std': 1.6591843366622925, 'epsilon_dpo/beta_margin_grad_mean': -0.1637529879808426, 'epsilon_dpo/beta_margin_grad_std': 0.16410239040851593, 'kl/beta': 0.01681213080883026, 'kl/avg_steps': 0.875, 'epoch': 0.35}
|
|||
|
|
35%|███████████████████████████████████████▌ | 236/681 [15:45<18:40, 2.52s/it]
35%|███████████████████████████████████████▋ | 237/681 [15:48<19:04, 2.58s/it]
{'loss': 0.3225, 'grad_norm': 47.88125991821289, 'learning_rate': 4.136269950853473e-07, 'rewards/chosen': -1.198293924331665, 'rewards/rejected': -3.849404811859131, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.651111125946045, 'logps/chosen': -99.72041320800781, 'logps/rejected': -334.0084228515625, 'logps/ref_chosen': -27.259479522705078, 'logps/ref_rejected': -100.87033081054688, 'logits/chosen': -2.0394325256347656, 'logits/rejected': -2.007416248321533, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.01652212254703045, 'epsilon_dpo/loss_margin_mean': 160.67715454101562, 'epsilon_dpo/beta_margin_mean': 2.651111125946045, 'epsilon_dpo/beta_margin_std': 1.6006019115447998, 'epsilon_dpo/beta_margin_grad_mean': -0.13068081438541412, 'epsilon_dpo/beta_margin_grad_std': 0.15401048958301544, 'kl/beta': 0.01666630059480667, 'kl/avg_steps': 0.875, 'epoch': 0.35}
|
|||
|
|
35%|███████████████████████████████████████▋ | 237/681 [15:48<19:04, 2.58s/it]
35%|███████████████████████████████████████▊ | 238/681 [15:50<19:05, 2.59s/it]
{'loss': 0.4646, 'grad_norm': 65.86769104003906, 'learning_rate': 4.126545750510605e-07, 'rewards/chosen': -1.598804235458374, 'rewards/rejected': -3.772151231765747, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.173346996307373, 'logps/chosen': -137.6107177734375, 'logps/rejected': -323.8140563964844, 'logps/ref_chosen': -40.190574645996094, 'logps/ref_rejected': -93.49894714355469, 'logits/chosen': -2.0702290534973145, 'logits/rejected': -2.123429298400879, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.016383972018957138, 'epsilon_dpo/loss_margin_mean': 132.8949737548828, 'epsilon_dpo/beta_margin_mean': 2.173346996307373, 'epsilon_dpo/beta_margin_std': 1.7011436223983765, 'epsilon_dpo/beta_margin_grad_mean': -0.1836518496274948, 'epsilon_dpo/beta_margin_grad_std': 0.17506077885627747, 'kl/beta': 0.01652173511683941, 'kl/avg_steps': 0.84375, 'epoch': 0.35}
|
|||
|
|
35%|███████████████████████████████████████▊ | 238/681 [15:50<19:05, 2.59s/it]
35%|████████████████████████████████████████ | 239/681 [15:53<19:02, 2.59s/it]
{'loss': 0.4034, 'grad_norm': 56.121368408203125, 'learning_rate': 4.116778689174514e-07, 'rewards/chosen': -1.4062585830688477, 'rewards/rejected': -3.755234956741333, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.3489763736724854, 'logps/chosen': -124.91606140136719, 'logps/rejected': -331.0585021972656, 'logps/ref_chosen': -38.33943176269531, 'logps/ref_rejected': -99.64225769042969, 'logits/chosen': -2.0602469444274902, 'logits/rejected': -1.9942574501037598, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.016236647963523865, 'epsilon_dpo/loss_margin_mean': 144.83961486816406, 'epsilon_dpo/beta_margin_mean': 2.3489763736724854, 'epsilon_dpo/beta_margin_std': 1.5995773077011108, 'epsilon_dpo/beta_margin_grad_mean': -0.16019320487976074, 'epsilon_dpo/beta_margin_grad_std': 0.1673043966293335, 'kl/beta': 0.016383498907089233, 'kl/avg_steps': 0.90625, 'epoch': 0.35}
|
|||
|
|
35%|████████████████████████████████████████ | 239/681 [15:53<19:02, 2.59s/it]
35%|████████████████████████████████████████▏ | 240/681 [15:55<18:39, 2.54s/it]
{'loss': 0.4095, 'grad_norm': 66.01368713378906, 'learning_rate': 4.106969024216348e-07, 'rewards/chosen': -1.2190780639648438, 'rewards/rejected': -3.4683375358581543, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.2492594718933105, 'logps/chosen': -111.68765258789062, 'logps/rejected': -295.5617980957031, 'logps/ref_chosen': -36.1579704284668, 'logps/ref_rejected': -80.07916259765625, 'logits/chosen': -2.0398552417755127, 'logits/rejected': -1.8625662326812744, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.016100972890853882, 'epsilon_dpo/loss_margin_mean': 139.95294189453125, 'epsilon_dpo/beta_margin_mean': 2.2492594718933105, 'epsilon_dpo/beta_margin_std': 1.4863446950912476, 'epsilon_dpo/beta_margin_grad_mean': -0.16005262732505798, 'epsilon_dpo/beta_margin_grad_std': 0.16643419861793518, 'kl/beta': 0.016236357390880585, 'kl/avg_steps': 0.84375, 'epoch': 0.35}
|
|||
|
|
35%|████████████████████████████████████████▏ | 240/681 [15:55<18:39, 2.54s/it]
35%|████████████████████████████████████████▎ | 241/681 [15:58<18:35, 2.53s/it]
{'loss': 0.426, 'grad_norm': 74.83716583251953, 'learning_rate': 4.097117014129903e-07, 'rewards/chosen': -1.3152108192443848, 'rewards/rejected': -3.7722537517547607, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.457043170928955, 'logps/chosen': -126.07571411132812, 'logps/rejected': -329.2753601074219, 'logps/ref_chosen': -44.0040397644043, 'logps/ref_rejected': -93.001220703125, 'logits/chosen': -2.196474313735962, 'logits/rejected': -1.913820505142212, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.015971289947628975, 'epsilon_dpo/loss_margin_mean': 154.20245361328125, 'epsilon_dpo/beta_margin_mean': 2.457042932510376, 'epsilon_dpo/beta_margin_std': 1.8046914339065552, 'epsilon_dpo/beta_margin_grad_mean': -0.16114287078380585, 'epsilon_dpo/beta_margin_grad_std': 0.18923403322696686, 'kl/beta': 0.016100509092211723, 'kl/avg_steps': 0.8125, 'epoch': 0.35}
|
|||
|
|
35%|████████████████████████████████████████▎ | 241/681 [15:58<18:35, 2.53s/it]
36%|████████████████████████████████████████▌ | 242/681 [16:00<18:37, 2.54s/it]
{'loss': 0.4175, 'grad_norm': 78.47781372070312, 'learning_rate': 4.087222918524807e-07, 'rewards/chosen': -1.2343041896820068, 'rewards/rejected': -3.691114902496338, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.45681095123291, 'logps/chosen': -109.82485961914062, 'logps/rejected': -324.636962890625, 'logps/ref_chosen': -32.014137268066406, 'logps/ref_rejected': -91.38673400878906, 'logits/chosen': -2.161656379699707, 'logits/rejected': -1.9027358293533325, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.015827594324946404, 'epsilon_dpo/loss_margin_mean': 155.43951416015625, 'epsilon_dpo/beta_margin_mean': 2.45681095123291, 'epsilon_dpo/beta_margin_std': 1.6646445989608765, 'epsilon_dpo/beta_margin_grad_mean': -0.14874504506587982, 'epsilon_dpo/beta_margin_grad_std': 0.17723040282726288, 'kl/beta': 0.0159707460552454, 'kl/avg_steps': 0.90625, 'epoch': 0.36}
|
|||
|
|
36%|████████████████████████████████████████▌ | 242/681 [16:00<18:37, 2.54s/it]
36%|████████████████████████████████████████▋ | 243/681 [16:03<18:30, 2.53s/it]
{'loss': 0.3675, 'grad_norm': 58.3444709777832, 'learning_rate': 4.07728699811968e-07, 'rewards/chosen': -1.282745122909546, 'rewards/rejected': -3.6751694679260254, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.3924245834350586, 'logps/chosen': -119.57041931152344, 'logps/rejected': -316.1097717285156, 'logps/ref_chosen': -37.93376922607422, 'logps/ref_rejected': -81.79078674316406, 'logits/chosen': -2.0235860347747803, 'logits/rejected': -1.7727184295654297, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.01569039188325405, 'epsilon_dpo/loss_margin_mean': 152.6823272705078, 'epsilon_dpo/beta_margin_mean': 2.3924245834350586, 'epsilon_dpo/beta_margin_std': 1.4725757837295532, 'epsilon_dpo/beta_margin_grad_mean': -0.14957940578460693, 'epsilon_dpo/beta_margin_grad_std': 0.1580885350704193, 'kl/beta': 0.01582731120288372, 'kl/avg_steps': 0.875, 'epoch': 0.36}
|
|||
|
|
36%|████████████████████████████████████████▋ | 243/681 [16:03<18:30, 2.53s/it]
36%|████████████████████████████████████████▊ | 244/681 [16:05<18:26, 2.53s/it]
{'loss': 0.4834, 'grad_norm': 60.811737060546875, 'learning_rate': 4.067309514735267e-07, 'rewards/chosen': -1.3352429866790771, 'rewards/rejected': -3.4218435287475586, 'rewards/accuracies': 0.890625, 'rewards/margins': 2.0866003036499023, 'logps/chosen': -125.60481262207031, 'logps/rejected': -321.4036865234375, 'logps/ref_chosen': -39.915008544921875, 'logps/ref_rejected': -101.33251953125, 'logits/chosen': -2.1224303245544434, 'logits/rejected': -2.0141685009002686, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.015569004230201244, 'epsilon_dpo/loss_margin_mean': 134.38136291503906, 'epsilon_dpo/beta_margin_mean': 2.0866005420684814, 'epsilon_dpo/beta_margin_std': 1.515892744064331, 'epsilon_dpo/beta_margin_grad_mean': -0.18348611891269684, 'epsilon_dpo/beta_margin_grad_std': 0.19074872136116028, 'kl/beta': 0.015690024942159653, 'kl/avg_steps': 0.78125, 'epoch': 0.36}
|
|||
|
|
36%|████████████████████████████████████████▊ | 244/681 [16:05<18:26, 2.53s/it]
36%|█████████████████████████████████████████ | 245/681 [16:08<18:21, 2.53s/it]
{'loss': 0.538, 'grad_norm': 70.79053497314453, 'learning_rate': 4.057290731287531e-07, 'rewards/chosen': -1.1700770854949951, 'rewards/rejected': -3.351145029067993, 'rewards/accuracies': 0.875, 'rewards/margins': 2.181067943572998, 'logps/chosen': -115.84690856933594, 'logps/rejected': -310.2763671875, 'logps/ref_chosen': -40.404693603515625, 'logps/ref_rejected': -93.24897766113281, 'logits/chosen': -2.187222957611084, 'logits/rejected': -1.977565050125122, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.015458044596016407, 'epsilon_dpo/loss_margin_mean': 141.5851593017578, 'epsilon_dpo/beta_margin_mean': 2.181067943572998, 'epsilon_dpo/beta_margin_std': 1.7734534740447998, 'epsilon_dpo/beta_margin_grad_mean': -0.19498883187770844, 'epsilon_dpo/beta_margin_grad_std': 0.21323725581169128, 'kl/beta': 0.01556839607656002, 'kl/avg_steps': 0.71875, 'epoch': 0.36}
|
|||
|
|
36%|█████████████████████████████████████████ | 245/681 [16:08<18:21, 2.53s/it]
36%|█████████████████████████████████████████▏ | 246/681 [16:10<18:24, 2.54s/it]
{'loss': 0.4609, 'grad_norm': 59.1838264465332, 'learning_rate': 4.047230911780736e-07, 'rewards/chosen': -1.3602585792541504, 'rewards/rejected': -3.4846174716949463, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.124358892440796, 'logps/chosen': -134.75564575195312, 'logps/rejected': -317.5809326171875, 'logps/ref_chosen': -46.212432861328125, 'logps/ref_rejected': -90.18721008300781, 'logits/chosen': -2.1199212074279785, 'logits/rejected': -1.9131314754486084, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.015333239920437336, 'epsilon_dpo/loss_margin_mean': 138.85049438476562, 'epsilon_dpo/beta_margin_mean': 2.124358892440796, 'epsilon_dpo/beta_margin_std': 1.5861018896102905, 'epsilon_dpo/beta_margin_grad_mean': -0.18123117089271545, 'epsilon_dpo/beta_margin_grad_std': 0.17709243297576904, 'kl/beta': 0.01545729674398899, 'kl/avg_steps': 0.8125, 'epoch': 0.36}
|
|||
|
|
36%|█████████████████████████████████████████▏ | 246/681 [16:10<18:24, 2.54s/it]
36%|█████████████████████████████████████████▎ | 247/681 [16:13<17:50, 2.47s/it]
{'loss': 0.4948, 'grad_norm': 73.05105590820312, 'learning_rate': 4.0371303213004814e-07, 'rewards/chosen': -1.479392409324646, 'rewards/rejected': -3.8148579597473145, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.335465431213379, 'logps/chosen': -138.3466796875, 'logps/rejected': -362.81231689453125, 'logps/ref_chosen': -41.23990249633789, 'logps/ref_rejected': -111.74742126464844, 'logits/chosen': -2.0222108364105225, 'logits/rejected': -2.00075626373291, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.015209660865366459, 'epsilon_dpo/loss_margin_mean': 153.95811462402344, 'epsilon_dpo/beta_margin_mean': 2.335465669631958, 'epsilon_dpo/beta_margin_std': 1.7916253805160522, 'epsilon_dpo/beta_margin_grad_mean': -0.16919678449630737, 'epsilon_dpo/beta_margin_grad_std': 0.20299823582172394, 'kl/beta': 0.015332718379795551, 'kl/avg_steps': 0.8125, 'epoch': 0.36}
|
|||
|
|
36%|█████████████████████████████████████████▎ | 247/681 [16:13<17:50, 2.47s/it]
36%|█████████████████████████████████████████▌ | 248/681 [16:15<17:44, 2.46s/it]
{'loss': 0.2904, 'grad_norm': 40.315460205078125, 'learning_rate': 4.0269892260067197e-07, 'rewards/chosen': -1.1554369926452637, 'rewards/rejected': -3.61720609664917, 'rewards/accuracies': 0.984375, 'rewards/margins': 2.461768865585327, 'logps/chosen': -102.28887939453125, 'logps/rejected': -337.76348876953125, 'logps/ref_chosen': -25.618247985839844, 'logps/ref_rejected': -97.59014892578125, 'logits/chosen': -1.9299895763397217, 'logits/rejected': -2.0007243156433105, 'kl/p_epsilon_steps': 0.984375, 'kl/n_epsilon_steps': 0.015625, 'epsilon_dpo/beta': 0.015063311904668808, 'epsilon_dpo/loss_margin_mean': 163.502685546875, 'epsilon_dpo/beta_margin_mean': 2.461768865585327, 'epsilon_dpo/beta_margin_std': 1.234181523323059, 'epsilon_dpo/beta_margin_grad_mean': -0.12139809876680374, 'epsilon_dpo/beta_margin_grad_std': 0.12686574459075928, 'kl/beta': 0.015209143981337547, 'kl/avg_steps': 0.96875, 'epoch': 0.36}
|
|||
|
|
36%|█████████████████████████████████████████▌ | 248/681 [16:15<17:44, 2.46s/it]
37%|█████████████████████████████████████████▋ | 249/681 [16:18<18:18, 2.54s/it]
{'loss': 0.4484, 'grad_norm': 58.40861129760742, 'learning_rate': 4.0168078931267426e-07, 'rewards/chosen': -1.377791404724121, 'rewards/rejected': -3.506488800048828, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.128697395324707, 'logps/chosen': -134.6277618408203, 'logps/rejected': -322.1723937988281, 'logps/ref_chosen': -42.576805114746094, 'logps/ref_rejected': -87.38154602050781, 'logits/chosen': -2.040287494659424, 'logits/rejected': -1.8837002515792847, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.014947031624615192, 'epsilon_dpo/loss_margin_mean': 142.73989868164062, 'epsilon_dpo/beta_margin_mean': 2.128697395324707, 'epsilon_dpo/beta_margin_std': 1.604970097541809, 'epsilon_dpo/beta_margin_grad_mean': -0.18095283210277557, 'epsilon_dpo/beta_margin_grad_std': 0.164636492729187, 'kl/beta': 0.015063218772411346, 'kl/avg_steps': 0.78125, 'epoch': 0.37}
|
|||
|
|
37%|█████████████████████████████████████████▋ | 249/681 [16:18<18:18, 2.54s/it]
37%|█████████████████████████████████████████▊ | 250/681 [16:20<18:06, 2.52s/it]
{'loss': 0.4175, 'grad_norm': 54.212379455566406, 'learning_rate': 4.006586590948141e-07, 'rewards/chosen': -1.2351545095443726, 'rewards/rejected': -3.3480517864227295, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.1128971576690674, 'logps/chosen': -126.69745635986328, 'logps/rejected': -306.03778076171875, 'logps/ref_chosen': -43.33977508544922, 'logps/ref_rejected': -79.84855651855469, 'logits/chosen': -2.1342391967773438, 'logits/rejected': -1.7132606506347656, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.014812478795647621, 'epsilon_dpo/loss_margin_mean': 142.83154296875, 'epsilon_dpo/beta_margin_mean': 2.1128971576690674, 'epsilon_dpo/beta_margin_std': 1.335999608039856, 'epsilon_dpo/beta_margin_grad_mean': -0.16431747376918793, 'epsilon_dpo/beta_margin_grad_std': 0.16239361464977264, 'kl/beta': 0.014946449548006058, 'kl/avg_steps': 0.90625, 'epoch': 0.37}
|
|||
|
|
37%|█████████████████████████████████████████▊ | 250/681 [16:20<18:06, 2.52s/it]
37%|██████████████████████████████████████████ | 251/681 [16:23<18:02, 2.52s/it]
{'loss': 0.4894, 'grad_norm': 58.540199279785156, 'learning_rate': 3.9963255888117325e-07, 'rewards/chosen': -1.332367181777954, 'rewards/rejected': -3.3120296001434326, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.979662299156189, 'logps/chosen': -128.45263671875, 'logps/rejected': -308.31683349609375, 'logps/ref_chosen': -37.8934211730957, 'logps/ref_rejected': -82.71955871582031, 'logits/chosen': -2.064459800720215, 'logits/rejected': -1.8460227251052856, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.014693334698677063, 'epsilon_dpo/loss_margin_mean': 135.0380859375, 'epsilon_dpo/beta_margin_mean': 1.979662299156189, 'epsilon_dpo/beta_margin_std': 1.4761016368865967, 'epsilon_dpo/beta_margin_grad_mean': -0.1908206194639206, 'epsilon_dpo/beta_margin_grad_std': 0.17475423216819763, 'kl/beta': 0.01481221430003643, 'kl/avg_steps': 0.8125, 'epoch': 0.37}
|
|||
|
|
37%|██████████████████████████████████████████ | 251/681 [16:23<18:02, 2.52s/it]
37%|██████████████████████████████████████████▏ | 252/681 [16:26<18:25, 2.58s/it]
{'loss': 0.4052, 'grad_norm': 61.02168273925781, 'learning_rate': 3.9860251571044666e-07, 'rewards/chosen': -1.3763771057128906, 'rewards/rejected': -3.5437517166137695, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.167374610900879, 'logps/chosen': -143.55337524414062, 'logps/rejected': -335.2132568359375, 'logps/ref_chosen': -49.172019958496094, 'logps/ref_rejected': -91.81843566894531, 'logits/chosen': -2.0972189903259277, 'logits/rejected': -1.8574435710906982, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.014561137184500694, 'epsilon_dpo/loss_margin_mean': 149.01345825195312, 'epsilon_dpo/beta_margin_mean': 2.167374849319458, 'epsilon_dpo/beta_margin_std': 1.4150327444076538, 'epsilon_dpo/beta_margin_grad_mean': -0.16394339501857758, 'epsilon_dpo/beta_margin_grad_std': 0.15933236479759216, 'kl/beta': 0.014692834578454494, 'kl/avg_steps': 0.90625, 'epoch': 0.37}
|
|||
|
|
37%|██████████████████████████████████████████▏ | 252/681 [16:26<18:25, 2.58s/it]
37%|██████████████████████████████████████████▎ | 253/681 [16:28<18:29, 2.59s/it]
{'loss': 0.5018, 'grad_norm': 65.06855773925781, 'learning_rate': 3.9756855672522986e-07, 'rewards/chosen': -1.4654229879379272, 'rewards/rejected': -3.4208884239196777, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.95546555519104, 'logps/chosen': -156.42469787597656, 'logps/rejected': -341.0745544433594, 'logps/ref_chosen': -55.18561553955078, 'logps/ref_rejected': -104.15153503417969, 'logits/chosen': -2.0035042762756348, 'logits/rejected': -2.029510021209717, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.014448565430939198, 'epsilon_dpo/loss_margin_mean': 135.68392944335938, 'epsilon_dpo/beta_margin_mean': 1.9554654359817505, 'epsilon_dpo/beta_margin_std': 1.4330785274505615, 'epsilon_dpo/beta_margin_grad_mean': -0.1948602944612503, 'epsilon_dpo/beta_margin_grad_std': 0.1849391907453537, 'kl/beta': 0.014560877345502377, 'kl/avg_steps': 0.78125, 'epoch': 0.37}
|
|||
|
|
37%|██████████████████████████████████████████▎ | 253/681 [16:28<18:29, 2.59s/it]
37%|██████████████████████████████████████████▌ | 254/681 [16:31<18:25, 2.59s/it]
{'loss': 0.5136, 'grad_norm': 57.888275146484375, 'learning_rate': 3.965307091713037e-07, 'rewards/chosen': -1.277719259262085, 'rewards/rejected': -3.309704065322876, 'rewards/accuracies': 0.859375, 'rewards/margins': 2.031984806060791, 'logps/chosen': -133.6912078857422, 'logps/rejected': -324.47515869140625, 'logps/ref_chosen': -44.80467224121094, 'logps/ref_rejected': -93.50021362304688, 'logits/chosen': -2.0362446308135986, 'logits/rejected': -1.8677754402160645, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.014350106939673424, 'epsilon_dpo/loss_margin_mean': 142.08840942382812, 'epsilon_dpo/beta_margin_mean': 2.031984806060791, 'epsilon_dpo/beta_margin_std': 1.5720576047897339, 'epsilon_dpo/beta_margin_grad_mean': -0.19600307941436768, 'epsilon_dpo/beta_margin_grad_std': 0.19512991607189178, 'kl/beta': 0.014448001980781555, 'kl/avg_steps': 0.6875, 'epoch': 0.37}
|
|||
|
|
37%|██████████████████████████████████████████▌ | 254/681 [16:31<18:25, 2.59s/it]
37%|██████████████████████████████████████████▋ | 255/681 [16:33<18:12, 2.57s/it]
{'loss': 0.4448, 'grad_norm': 61.83622360229492, 'learning_rate': 3.954890003969163e-07, 'rewards/chosen': -1.3225433826446533, 'rewards/rejected': -3.631333112716675, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.3087897300720215, 'logps/chosen': -130.06533813476562, 'logps/rejected': -352.3484191894531, 'logps/ref_chosen': -37.239234924316406, 'logps/ref_rejected': -96.95054626464844, 'logits/chosen': -1.7996987104415894, 'logits/rejected': -1.752530574798584, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.01423418615013361, 'epsilon_dpo/loss_margin_mean': 162.57176208496094, 'epsilon_dpo/beta_margin_mean': 2.3087897300720215, 'epsilon_dpo/beta_margin_std': 1.6275063753128052, 'epsilon_dpo/beta_margin_grad_mean': -0.16781477630138397, 'epsilon_dpo/beta_margin_grad_std': 0.1848216950893402, 'kl/beta': 0.014349350705742836, 'kl/avg_steps': 0.8125, 'epoch': 0.37}
|
|||
|
|
37%|██████████████████████████████████████████▋ | 255/681 [16:33<18:12, 2.57s/it]
38%|██████████████████████████████████████████▊ | 256/681 [16:36<17:37, 2.49s/it]
{'loss': 0.3517, 'grad_norm': 47.72064208984375, 'learning_rate': 3.944434578520628e-07, 'rewards/chosen': -1.1789929866790771, 'rewards/rejected': -3.5271737575531006, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.3481807708740234, 'logps/chosen': -118.48297119140625, 'logps/rejected': -349.3486328125, 'logps/ref_chosen': -35.025508880615234, 'logps/ref_rejected': -99.25279998779297, 'logits/chosen': -1.8363198041915894, 'logits/rejected': -1.8857228755950928, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.014110567979514599, 'epsilon_dpo/loss_margin_mean': 166.6383819580078, 'epsilon_dpo/beta_margin_mean': 2.3481807708740234, 'epsilon_dpo/beta_margin_std': 1.3911449909210205, 'epsilon_dpo/beta_margin_grad_mean': -0.14517581462860107, 'epsilon_dpo/beta_margin_grad_std': 0.1478438377380371, 'kl/beta': 0.01423370186239481, 'kl/avg_steps': 0.875, 'epoch': 0.38}
|
|||
|
|
38%|██████████████████████████████████████████▊ | 256/681 [16:36<17:37, 2.49s/it]
38%|███████████████████████████████████████████ | 257/681 [16:38<17:50, 2.52s/it]
{'loss': 0.4202, 'grad_norm': 58.855709075927734, 'learning_rate': 3.933941090877615e-07, 'rewards/chosen': -1.2731714248657227, 'rewards/rejected': -3.4311952590942383, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.1580240726470947, 'logps/chosen': -125.66242980957031, 'logps/rejected': -329.75701904296875, 'logps/ref_chosen': -34.74375534057617, 'logps/ref_rejected': -84.338134765625, 'logits/chosen': -1.8602499961853027, 'logits/rejected': -1.7255544662475586, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.013992580585181713, 'epsilon_dpo/loss_margin_mean': 154.5001983642578, 'epsilon_dpo/beta_margin_mean': 2.1580240726470947, 'epsilon_dpo/beta_margin_std': 1.4499742984771729, 'epsilon_dpo/beta_margin_grad_mean': -0.1678382307291031, 'epsilon_dpo/beta_margin_grad_std': 0.16572539508342743, 'kl/beta': 0.014110236428678036, 'kl/avg_steps': 0.84375, 'epoch': 0.38}
|
|||
|
|
38%|███████████████████████████████████████████ | 257/681 [16:38<17:50, 2.52s/it]
38%|███████████████████████████████████████████▏ | 258/681 [16:41<17:31, 2.49s/it]
{'loss': 0.4481, 'grad_norm': 56.01180648803711, 'learning_rate': 3.923409817553284e-07, 'rewards/chosen': -1.2534571886062622, 'rewards/rejected': -3.497140884399414, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.2436838150024414, 'logps/chosen': -128.53773498535156, 'logps/rejected': -356.6092529296875, 'logps/ref_chosen': -38.32011032104492, 'logps/ref_rejected': -104.34061431884766, 'logits/chosen': -1.829561710357666, 'logits/rejected': -1.8402704000473022, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.013875506818294525, 'epsilon_dpo/loss_margin_mean': 162.05101013183594, 'epsilon_dpo/beta_margin_mean': 2.2436838150024414, 'epsilon_dpo/beta_margin_std': 1.5077780485153198, 'epsilon_dpo/beta_margin_grad_mean': -0.16494029760360718, 'epsilon_dpo/beta_margin_grad_std': 0.19155991077423096, 'kl/beta': 0.01399217825382948, 'kl/avg_steps': 0.84375, 'epoch': 0.38}
|
|||
|
|
38%|███████████████████████████████████████████▏ | 258/681 [16:41<17:31, 2.49s/it]
38%|███████████████████████████████████████████▎ | 259/681 [16:43<17:29, 2.49s/it]
{'loss': 0.4435, 'grad_norm': 53.66272735595703, 'learning_rate': 3.9128410360564793e-07, 'rewards/chosen': -1.3125896453857422, 'rewards/rejected': -3.3289833068847656, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.0163938999176025, 'logps/chosen': -133.28622436523438, 'logps/rejected': -336.7703552246094, 'logps/ref_chosen': -37.96626663208008, 'logps/ref_rejected': -94.62816619873047, 'logits/chosen': -1.766379952430725, 'logits/rejected': -1.7046854496002197, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.013755074702203274, 'epsilon_dpo/loss_margin_mean': 146.82223510742188, 'epsilon_dpo/beta_margin_mean': 2.0163936614990234, 'epsilon_dpo/beta_margin_std': 1.3358848094940186, 'epsilon_dpo/beta_margin_grad_mean': -0.17741423845291138, 'epsilon_dpo/beta_margin_grad_std': 0.16296601295471191, 'kl/beta': 0.01387510634958744, 'kl/avg_steps': 0.875, 'epoch': 0.38}
|
|||
|
|
38%|███████████████████████████████████████████▎ | 259/681 [16:43<17:29, 2.49s/it]
38%|███████████████████████████████████████████▌ | 260/681 [16:46<17:39, 2.52s/it]
{'loss': 0.4876, 'grad_norm': 71.4144287109375, 'learning_rate': 3.9022350248844246e-07, 'rewards/chosen': -1.3459320068359375, 'rewards/rejected': -3.411945343017578, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.0660133361816406, 'logps/chosen': -128.06361389160156, 'logps/rejected': -351.5653076171875, 'logps/ref_chosen': -29.670434951782227, 'logps/ref_rejected': -101.38003540039062, 'logits/chosen': -1.7605104446411133, 'logits/rejected': -1.844254732131958, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.01364436000585556, 'epsilon_dpo/loss_margin_mean': 151.79209899902344, 'epsilon_dpo/beta_margin_mean': 2.0660133361816406, 'epsilon_dpo/beta_margin_std': 1.5317002534866333, 'epsilon_dpo/beta_margin_grad_mean': -0.18185892701148987, 'epsilon_dpo/beta_margin_grad_std': 0.1839314103126526, 'kl/beta': 0.013754752464592457, 'kl/avg_steps': 0.8125, 'epoch': 0.38}
|
|||
|
|
38%|███████████████████████████████████████████▌ | 260/681 [16:46<17:39, 2.52s/it]
38%|███████████████████████████████████████████▋ | 261/681 [16:48<17:34, 2.51s/it]
{'loss': 0.396, 'grad_norm': 66.16899871826172, 'learning_rate': 3.891592063515376e-07, 'rewards/chosen': -1.1082427501678467, 'rewards/rejected': -3.654106855392456, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.5458641052246094, 'logps/chosen': -113.61106872558594, 'logps/rejected': -365.1859436035156, 'logps/ref_chosen': -31.892623901367188, 'logps/ref_rejected': -94.94419860839844, 'logits/chosen': -1.7042864561080933, 'logits/rejected': -1.5886824131011963, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.013534392230212688, 'epsilon_dpo/loss_margin_mean': 188.52330017089844, 'epsilon_dpo/beta_margin_mean': 2.5458641052246094, 'epsilon_dpo/beta_margin_std': 1.7510801553726196, 'epsilon_dpo/beta_margin_grad_mean': -0.14695821702480316, 'epsilon_dpo/beta_margin_grad_std': 0.183539479970932, 'kl/beta': 0.013643896207213402, 'kl/avg_steps': 0.8125, 'epoch': 0.38}
|
|||
|
|
38%|███████████████████████████████████████████▋ | 261/681 [16:48<17:34, 2.51s/it]
38%|███████████████████████████████████████████▊ | 262/681 [16:51<17:15, 2.47s/it]
{'loss': 0.4156, 'grad_norm': 56.77939224243164, 'learning_rate': 3.880912432401264e-07, 'rewards/chosen': -1.348780632019043, 'rewards/rejected': -3.3994228839874268, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.050642251968384, 'logps/chosen': -134.6734619140625, 'logps/rejected': -343.780029296875, 'logps/ref_chosen': -34.26129913330078, 'logps/ref_rejected': -90.29788208007812, 'logits/chosen': -1.6830520629882812, 'logits/rejected': -1.5291244983673096, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.013416852802038193, 'epsilon_dpo/loss_margin_mean': 153.06997680664062, 'epsilon_dpo/beta_margin_mean': 2.050642251968384, 'epsilon_dpo/beta_margin_std': 1.2614741325378418, 'epsilon_dpo/beta_margin_grad_mean': -0.16779224574565887, 'epsilon_dpo/beta_margin_grad_std': 0.15692253410816193, 'kl/beta': 0.013533933088183403, 'kl/avg_steps': 0.875, 'epoch': 0.38}
|
|||
|
|
38%|███████████████████████████████████████████▊ | 262/681 [16:51<17:15, 2.47s/it]
39%|████████████████████████████████████████████ | 263/681 [16:53<17:39, 2.53s/it]
{'loss': 0.3749, 'grad_norm': 59.517337799072266, 'learning_rate': 3.870196412960302e-07, 'rewards/chosen': -1.233377456665039, 'rewards/rejected': -3.5795066356658936, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.3461294174194336, 'logps/chosen': -131.9771728515625, 'logps/rejected': -371.615966796875, 'logps/ref_chosen': -39.4767951965332, 'logps/ref_rejected': -102.44886779785156, 'logits/chosen': -1.8858226537704468, 'logits/rejected': -1.5893621444702148, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.013304665684700012, 'epsilon_dpo/loss_margin_mean': 176.666748046875, 'epsilon_dpo/beta_margin_mean': 2.3461291790008545, 'epsilon_dpo/beta_margin_std': 1.4775606393814087, 'epsilon_dpo/beta_margin_grad_mean': -0.14855755865573883, 'epsilon_dpo/beta_margin_grad_std': 0.16537067294120789, 'kl/beta': 0.013416538015007973, 'kl/avg_steps': 0.84375, 'epoch': 0.39}
|
|||
|
|
39%|████████████████████████████████████████████ | 263/681 [16:53<17:39, 2.53s/it]
39%|████████████████████████████████████████████▏ | 264/681 [16:56<17:53, 2.58s/it]
{'loss': 0.3936, 'grad_norm': 60.96957015991211, 'learning_rate': 3.8594442875695665e-07, 'rewards/chosen': -1.1168205738067627, 'rewards/rejected': -3.4131574630737305, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.296337127685547, 'logps/chosen': -123.23445129394531, 'logps/rejected': -359.046875, 'logps/ref_chosen': -38.707645416259766, 'logps/ref_rejected': -100.15180969238281, 'logits/chosen': -1.6896053552627563, 'logits/rejected': -1.7085572481155396, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.01318918913602829, 'epsilon_dpo/loss_margin_mean': 174.36825561523438, 'epsilon_dpo/beta_margin_mean': 2.2963368892669678, 'epsilon_dpo/beta_margin_std': 1.5442652702331543, 'epsilon_dpo/beta_margin_grad_mean': -0.15769457817077637, 'epsilon_dpo/beta_margin_grad_std': 0.15815286338329315, 'kl/beta': 0.013304282911121845, 'kl/avg_steps': 0.875, 'epoch': 0.39}
|
|||
|
|
39%|████████████████████████████████████████████▏ | 264/681 [16:56<17:53, 2.58s/it]
39%|████████████████████████████████████████████▎ | 265/681 [16:58<17:35, 2.54s/it]
{'loss': 0.4478, 'grad_norm': 61.71824645996094, 'learning_rate': 3.848656339557562e-07, 'rewards/chosen': -1.4101543426513672, 'rewards/rejected': -3.5063629150390625, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.0962088108062744, 'logps/chosen': -145.28750610351562, 'logps/rejected': -362.984619140625, 'logps/ref_chosen': -37.51420593261719, 'logps/ref_rejected': -94.66896057128906, 'logits/chosen': -1.6642320156097412, 'logits/rejected': -1.5806505680084229, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.013070664368569851, 'epsilon_dpo/loss_margin_mean': 160.54234313964844, 'epsilon_dpo/beta_margin_mean': 2.0962088108062744, 'epsilon_dpo/beta_margin_std': 1.5082602500915527, 'epsilon_dpo/beta_margin_grad_mean': -0.17950545251369476, 'epsilon_dpo/beta_margin_grad_std': 0.168075293302536, 'kl/beta': 0.013188880868256092, 'kl/avg_steps': 0.90625, 'epoch': 0.39}
|
|||
|
|
39%|████████████████████████████████████████████▎ | 265/681 [16:58<17:35, 2.54s/it]
39%|████████████████████████████████████████████▌ | 266/681 [17:01<17:39, 2.55s/it]
{'loss': 0.5442, 'grad_norm': 68.82777404785156, 'learning_rate': 3.8378328531967507e-07, 'rewards/chosen': -1.5156571865081787, 'rewards/rejected': -3.358886241912842, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.843229055404663, 'logps/chosen': -167.84719848632812, 'logps/rejected': -332.4281311035156, 'logps/ref_chosen': -51.23357391357422, 'logps/ref_rejected': -73.29232025146484, 'logits/chosen': -1.6910204887390137, 'logits/rejected': -1.2894883155822754, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.012973698787391186, 'epsilon_dpo/loss_margin_mean': 142.52218627929688, 'epsilon_dpo/beta_margin_mean': 1.843229055404663, 'epsilon_dpo/beta_margin_std': 1.3907108306884766, 'epsilon_dpo/beta_margin_grad_mean': -0.20105549693107605, 'epsilon_dpo/beta_margin_grad_std': 0.19386924803256989, 'kl/beta': 0.013070429675281048, 'kl/avg_steps': 0.75, 'epoch': 0.39}
|
|||
|
|
39%|████████████████████████████████████████████▌ | 266/681 [17:01<17:39, 2.55s/it]
39%|████████████████████████████████████████████▋ | 267/681 [17:03<17:29, 2.54s/it]
{'loss': 0.4993, 'grad_norm': 67.35930633544922, 'learning_rate': 3.8269741136960646e-07, 'rewards/chosen': -1.3415794372558594, 'rewards/rejected': -3.271149158477783, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.9295697212219238, 'logps/chosen': -154.36102294921875, 'logps/rejected': -349.48822021484375, 'logps/ref_chosen': -50.439453125, 'logps/ref_rejected': -95.28913879394531, 'logits/chosen': -1.8555903434753418, 'logits/rejected': -1.507951021194458, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.012881174683570862, 'epsilon_dpo/loss_margin_mean': 150.2775115966797, 'epsilon_dpo/beta_margin_mean': 1.9295697212219238, 'epsilon_dpo/beta_margin_std': 1.4559646844863892, 'epsilon_dpo/beta_margin_grad_mean': -0.19470059871673584, 'epsilon_dpo/beta_margin_grad_std': 0.1797318160533905, 'kl/beta': 0.012973131611943245, 'kl/avg_steps': 0.71875, 'epoch': 0.39}
|
|||
|
|
39%|████████████████████████████████████████████▋ | 267/681 [17:04<17:29, 2.54s/it]
39%|████████████████████████████████████████████▊ | 268/681 [17:06<17:23, 2.53s/it]
{'loss': 0.5814, 'grad_norm': 80.12260437011719, 'learning_rate': 3.8160804071933894e-07, 'rewards/chosen': -1.461806058883667, 'rewards/rejected': -3.4406800270080566, 'rewards/accuracies': 0.859375, 'rewards/margins': 1.9788737297058105, 'logps/chosen': -156.72665405273438, 'logps/rejected': -377.4356384277344, 'logps/ref_chosen': -42.84587097167969, 'logps/ref_rejected': -108.26278686523438, 'logits/chosen': -1.6909327507019043, 'logits/rejected': -1.6416935920715332, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.012789253145456314, 'epsilon_dpo/loss_margin_mean': 155.29205322265625, 'epsilon_dpo/beta_margin_mean': 1.9788737297058105, 'epsilon_dpo/beta_margin_std': 1.7514647245407104, 'epsilon_dpo/beta_margin_grad_mean': -0.21375124156475067, 'epsilon_dpo/beta_margin_grad_std': 0.20960654318332672, 'kl/beta': 0.012880552560091019, 'kl/avg_steps': 0.71875, 'epoch': 0.39}
|
|||
|
|
39%|████████████████████████████████████████████▊ | 268/681 [17:06<17:23, 2.53s/it]
40%|█████████████████████████████████████████████ | 269/681 [17:09<17:20, 2.52s/it]
{'loss': 0.3916, 'grad_norm': 59.1228141784668, 'learning_rate': 3.8051520207480204e-07, 'rewards/chosen': -1.253930926322937, 'rewards/rejected': -3.7381930351257324, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.484261989593506, 'logps/chosen': -142.43875122070312, 'logps/rejected': -410.3055419921875, 'logps/ref_chosen': -43.99567413330078, 'logps/ref_rejected': -115.62770080566406, 'logits/chosen': -1.7581638097763062, 'logits/rejected': -1.7033562660217285, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.012697985395789146, 'epsilon_dpo/loss_margin_mean': 196.23475646972656, 'epsilon_dpo/beta_margin_mean': 2.484262228012085, 'epsilon_dpo/beta_margin_std': 1.5870592594146729, 'epsilon_dpo/beta_margin_grad_mean': -0.15136900544166565, 'epsilon_dpo/beta_margin_grad_std': 0.18286660313606262, 'kl/beta': 0.012788633815944195, 'kl/avg_steps': 0.71875, 'epoch': 0.4}
|
|||
|
|
40%|█████████████████████████████████████████████ | 269/681 [17:09<17:20, 2.52s/it]
40%|█████████████████████████████████████████████▏ | 270/681 [17:11<17:29, 2.55s/it]
{'loss': 0.512, 'grad_norm': 79.74817657470703, 'learning_rate': 3.794189242333106e-07, 'rewards/chosen': -1.5177865028381348, 'rewards/rejected': -3.590897798538208, 'rewards/accuracies': 0.875, 'rewards/margins': 2.0731112957000732, 'logps/chosen': -164.27764892578125, 'logps/rejected': -401.3154296875, 'logps/ref_chosen': -44.127079010009766, 'logps/ref_rejected': -116.14840698242188, 'logits/chosen': -1.8194841146469116, 'logits/rejected': -1.6801855564117432, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.012599433772265911, 'epsilon_dpo/loss_margin_mean': 165.01644897460938, 'epsilon_dpo/beta_margin_mean': 2.0731112957000732, 'epsilon_dpo/beta_margin_std': 1.5216357707977295, 'epsilon_dpo/beta_margin_grad_mean': -0.18488836288452148, 'epsilon_dpo/beta_margin_grad_std': 0.2021288424730301, 'kl/beta': 0.012697371654212475, 'kl/avg_steps': 0.78125, 'epoch': 0.4}
|
|||
|
|
40%|█████████████████████████████████████████████▏ | 270/681 [17:11<17:29, 2.55s/it]
40%|█████████████████████████████████████████████▎ | 271/681 [17:14<17:13, 2.52s/it]
{'loss': 0.5335, 'grad_norm': 58.63235855102539, 'learning_rate': 3.7831923608280514e-07, 'rewards/chosen': -1.219379186630249, 'rewards/rejected': -3.2758355140686035, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.0564560890197754, 'logps/chosen': -133.10488891601562, 'logps/rejected': -360.34832763671875, 'logps/ref_chosen': -35.64381408691406, 'logps/ref_rejected': -97.9295883178711, 'logits/chosen': -1.7537434101104736, 'logits/rejected': -1.6001136302947998, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.012493887916207314, 'epsilon_dpo/loss_margin_mean': 164.9576873779297, 'epsilon_dpo/beta_margin_mean': 2.0564560890197754, 'epsilon_dpo/beta_margin_std': 1.6271862983703613, 'epsilon_dpo/beta_margin_grad_mean': -0.19335998594760895, 'epsilon_dpo/beta_margin_grad_std': 0.19798798859119415, 'kl/beta': 0.012598942033946514, 'kl/avg_steps': 0.84375, 'epoch': 0.4}
|
|||
|
|
40%|█████████████████████████████████████████████▎ | 271/681 [17:14<17:13, 2.52s/it]
40%|█████████████████████████████████████████████▌ | 272/681 [17:16<17:23, 2.55s/it]
{'loss': 0.336, 'grad_norm': 43.1225471496582, 'learning_rate': 3.772161666010912e-07, 'rewards/chosen': -1.065723180770874, 'rewards/rejected': -3.6643643379211426, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.5986409187316895, 'logps/chosen': -112.50562286376953, 'logps/rejected': -408.69091796875, 'logps/ref_chosen': -26.52655792236328, 'logps/ref_rejected': -112.5738525390625, 'logits/chosen': -1.6689045429229736, 'logits/rejected': -1.6010162830352783, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.012385448440909386, 'epsilon_dpo/loss_margin_mean': 210.13800048828125, 'epsilon_dpo/beta_margin_mean': 2.5986411571502686, 'epsilon_dpo/beta_margin_std': 1.48728609085083, 'epsilon_dpo/beta_margin_grad_mean': -0.12951304018497467, 'epsilon_dpo/beta_margin_grad_std': 0.1683792918920517, 'kl/beta': 0.012493528425693512, 'kl/avg_steps': 0.875, 'epoch': 0.4}
|
|||
|
|
40%|█████████████████████████████████████████████▌ | 272/681 [17:16<17:23, 2.55s/it]
40%|█████████████████████████████████████████████▋ | 273/681 [17:19<17:18, 2.54s/it]
{'loss': 0.3461, 'grad_norm': 50.07746505737305, 'learning_rate': 3.761097448550755e-07, 'rewards/chosen': -1.2572829723358154, 'rewards/rejected': -3.6104252338409424, 'rewards/accuracies': 0.984375, 'rewards/margins': 2.353142261505127, 'logps/chosen': -137.0894317626953, 'logps/rejected': -391.7444152832031, 'logps/ref_chosen': -34.63496017456055, 'logps/ref_rejected': -97.36636352539062, 'logits/chosen': -1.71974515914917, 'logits/rejected': -1.6216449737548828, 'kl/p_epsilon_steps': 0.984375, 'kl/n_epsilon_steps': 0.015625, 'epsilon_dpo/beta': 0.012266403064131737, 'epsilon_dpo/loss_margin_mean': 191.923583984375, 'epsilon_dpo/beta_margin_mean': 2.353142261505127, 'epsilon_dpo/beta_margin_std': 1.3905528783798218, 'epsilon_dpo/beta_margin_grad_mean': -0.14163939654827118, 'epsilon_dpo/beta_margin_grad_std': 0.14621266722679138, 'kl/beta': 0.012385157868266106, 'kl/avg_steps': 0.96875, 'epoch': 0.4}
|
|||
|
|
40%|█████████████████████████████████████████████▋ | 273/681 [17:19<17:18, 2.54s/it]
40%|█████████████████████████████████████████████▊ | 274/681 [17:21<17:41, 2.61s/it]
{'loss': 0.4745, 'grad_norm': 58.21686553955078, 'learning_rate': 3.75e-07, 'rewards/chosen': -1.4374797344207764, 'rewards/rejected': -3.5061697959899902, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.0686898231506348, 'logps/chosen': -150.64378356933594, 'logps/rejected': -371.6086730957031, 'logps/ref_chosen': -32.576805114746094, 'logps/ref_rejected': -83.13209533691406, 'logits/chosen': -1.5487819910049438, 'logits/rejected': -1.4224430322647095, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.012164046056568623, 'epsilon_dpo/loss_margin_mean': 170.40960693359375, 'epsilon_dpo/beta_margin_mean': 2.068690061569214, 'epsilon_dpo/beta_margin_std': 1.446864366531372, 'epsilon_dpo/beta_margin_grad_mean': -0.1828116923570633, 'epsilon_dpo/beta_margin_grad_std': 0.1835782825946808, 'kl/beta': 0.012266327627003193, 'kl/avg_steps': 0.84375, 'epoch': 0.4}
|
|||
|
|
40%|█████████████████████████████████████████████▊ | 274/681 [17:22<17:41, 2.61s/it]
40%|██████████████████████████████████████████████ | 275/681 [17:24<17:25, 2.58s/it]
{'loss': 0.5343, 'grad_norm': 58.6602783203125, 'learning_rate': 3.738869612786737e-07, 'rewards/chosen': -1.4348502159118652, 'rewards/rejected': -3.4032628536224365, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.9684127569198608, 'logps/chosen': -152.82989501953125, 'logps/rejected': -380.90191650390625, 'logps/ref_chosen': -34.1890983581543, 'logps/ref_rejected': -98.71599578857422, 'logits/chosen': -1.6947541236877441, 'logits/rejected': -1.5999658107757568, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.012066072784364223, 'epsilon_dpo/loss_margin_mean': 163.5451202392578, 'epsilon_dpo/beta_margin_mean': 1.9684127569198608, 'epsilon_dpo/beta_margin_std': 1.5511200428009033, 'epsilon_dpo/beta_margin_grad_mean': -0.1988518238067627, 'epsilon_dpo/beta_margin_grad_std': 0.19736242294311523, 'kl/beta': 0.012163696810603142, 'kl/avg_steps': 0.8125, 'epoch': 0.4}
|
|||
|
|
40%|██████████████████████████████████████████████ | 275/681 [17:24<17:25, 2.58s/it]
41%|██████████████████████████████████████████████▏ | 276/681 [17:27<17:19, 2.57s/it]
{'loss': 0.537, 'grad_norm': 62.463478088378906, 'learning_rate': 3.7277065802070204e-07, 'rewards/chosen': -1.6980412006378174, 'rewards/rejected': -3.442594289779663, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.7445529699325562, 'logps/chosen': -188.8147735595703, 'logps/rejected': -364.2151794433594, 'logps/ref_chosen': -47.06272888183594, 'logps/ref_rejected': -76.37776947021484, 'logits/chosen': -1.75152587890625, 'logits/rejected': -1.3374929428100586, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.011965055949985981, 'epsilon_dpo/loss_margin_mean': 146.08535766601562, 'epsilon_dpo/beta_margin_mean': 1.7445530891418457, 'epsilon_dpo/beta_margin_std': 1.3083961009979248, 'epsilon_dpo/beta_margin_grad_mean': -0.2083517462015152, 'epsilon_dpo/beta_margin_grad_std': 0.1770058423280716, 'kl/beta': 0.012065663002431393, 'kl/avg_steps': 0.84375, 'epoch': 0.41}
|
|||
|
|
41%|██████████████████████████████████████████████▏ | 276/681 [17:27<17:19, 2.57s/it]
41%|██████████████████████████████████████████████▎ | 277/681 [17:29<16:51, 2.50s/it]
{'loss': 0.4258, 'grad_norm': 52.0661735534668, 'learning_rate': 3.71651119641714e-07, 'rewards/chosen': -1.4755170345306396, 'rewards/rejected': -3.613009452819824, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.1374926567077637, 'logps/chosen': -159.37274169921875, 'logps/rejected': -404.24322509765625, 'logps/ref_chosen': -35.24298095703125, 'logps/ref_rejected': -99.66352844238281, 'logits/chosen': -1.622573971748352, 'logits/rejected': -1.471212387084961, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.01186494529247284, 'epsilon_dpo/loss_margin_mean': 180.449951171875, 'epsilon_dpo/beta_margin_mean': 2.1374924182891846, 'epsilon_dpo/beta_margin_std': 1.4325969219207764, 'epsilon_dpo/beta_margin_grad_mean': -0.17211416363716125, 'epsilon_dpo/beta_margin_grad_std': 0.16370359063148499, 'kl/beta': 0.011964711360633373, 'kl/avg_steps': 0.84375, 'epoch': 0.41}
|
|||
|
|
41%|██████████████████████████████████████████████▎ | 277/681 [17:29<16:51, 2.50s/it]
41%|██████████████████████████████████████████████▌ | 278/681 [17:31<16:57, 2.52s/it]
{'loss': 0.4658, 'grad_norm': 54.76714324951172, 'learning_rate': 3.705283756425872e-07, 'rewards/chosen': -1.5532996654510498, 'rewards/rejected': -3.5797531604766846, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.0264534950256348, 'logps/chosen': -169.8358154296875, 'logps/rejected': -400.4996337890625, 'logps/ref_chosen': -38.0869140625, 'logps/ref_rejected': -96.20486450195312, 'logits/chosen': -1.7193242311477661, 'logits/rejected': -1.5687487125396729, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.011769380420446396, 'epsilon_dpo/loss_margin_mean': 172.5458526611328, 'epsilon_dpo/beta_margin_mean': 2.0264534950256348, 'epsilon_dpo/beta_margin_std': 1.5373589992523193, 'epsilon_dpo/beta_margin_grad_mean': -0.18432171642780304, 'epsilon_dpo/beta_margin_grad_std': 0.17036166787147522, 'kl/beta': 0.011864603497087955, 'kl/avg_steps': 0.8125, 'epoch': 0.41}
|
|||
|
|
41%|██████████████████████████████████████████████▌ | 278/681 [17:31<16:57, 2.52s/it]
41%|██████████████████████████████████████████████▋ | 279/681 [17:34<16:29, 2.46s/it]
{'loss': 0.4424, 'grad_norm': 56.205360412597656, 'learning_rate': 3.6940245560867e-07, 'rewards/chosen': -1.4212896823883057, 'rewards/rejected': -3.5274763107299805, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.106186628341675, 'logps/chosen': -151.58514404296875, 'logps/rejected': -395.55865478515625, 'logps/ref_chosen': -29.908262252807617, 'logps/ref_rejected': -93.1087646484375, 'logits/chosen': -1.4648022651672363, 'logits/rejected': -1.4414713382720947, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.011667169630527496, 'epsilon_dpo/loss_margin_mean': 180.77297973632812, 'epsilon_dpo/beta_margin_mean': 2.106186628341675, 'epsilon_dpo/beta_margin_std': 1.458591341972351, 'epsilon_dpo/beta_margin_grad_mean': -0.17751751840114594, 'epsilon_dpo/beta_margin_grad_std': 0.16590261459350586, 'kl/beta': 0.011768980883061886, 'kl/avg_steps': 0.875, 'epoch': 0.41}
|
|||
|
|
41%|██████████████████████████████████████████████▋ | 279/681 [17:34<16:29, 2.46s/it]
41%|██████████████████████████████████████████████▊ | 280/681 [17:36<16:24, 2.46s/it]
{'loss': 0.417, 'grad_norm': 52.6536865234375, 'learning_rate': 3.6827338920900253e-07, 'rewards/chosen': -1.4572311639785767, 'rewards/rejected': -3.623271942138672, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.1660408973693848, 'logps/chosen': -165.87985229492188, 'logps/rejected': -418.8723449707031, 'logps/ref_chosen': -40.1248664855957, 'logps/ref_rejected': -105.53138732910156, 'logits/chosen': -1.6275845766067505, 'logits/rejected': -1.6037421226501465, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.01156961265951395, 'epsilon_dpo/loss_margin_mean': 187.58596801757812, 'epsilon_dpo/beta_margin_mean': 2.1660408973693848, 'epsilon_dpo/beta_margin_std': 1.4027117490768433, 'epsilon_dpo/beta_margin_grad_mean': -0.16350157558918, 'epsilon_dpo/beta_margin_grad_std': 0.16755622625350952, 'kl/beta': 0.011666894890367985, 'kl/avg_steps': 0.84375, 'epoch': 0.41}
|
|||
|
|
41%|██████████████████████████████████████████████▊ | 280/681 [17:36<16:24, 2.46s/it]
41%|███████████████████████████████████████████████ | 281/681 [17:39<16:09, 2.42s/it]
{'loss': 0.45, 'grad_norm': 54.2619743347168, 'learning_rate': 3.6714120619553435e-07, 'rewards/chosen': -1.4972405433654785, 'rewards/rejected': -3.5504188537597656, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.053178548812866, 'logps/chosen': -169.83157348632812, 'logps/rejected': -396.84686279296875, 'logps/ref_chosen': -39.501502990722656, 'logps/ref_rejected': -87.23008728027344, 'logits/chosen': -1.6719520092010498, 'logits/rejected': -1.367035150527954, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.011469194665551186, 'epsilon_dpo/loss_margin_mean': 179.28672790527344, 'epsilon_dpo/beta_margin_mean': 2.053178548812866, 'epsilon_dpo/beta_margin_std': 1.504626750946045, 'epsilon_dpo/beta_margin_grad_mean': -0.18068785965442657, 'epsilon_dpo/beta_margin_grad_std': 0.1649240106344223, 'kl/beta': 0.011569279246032238, 'kl/avg_steps': 0.875, 'epoch': 0.41}
|
|||
|
|
41%|███████████████████████████████████████████████ | 281/681 [17:39<16:09, 2.42s/it]
41%|███████████████████████████████████████████████▏ | 282/681 [17:41<16:16, 2.45s/it]
{'loss': 0.4842, 'grad_norm': 54.99889373779297, 'learning_rate': 3.660059364023408e-07, 'rewards/chosen': -1.4642865657806396, 'rewards/rejected': -3.4173216819763184, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.9530349969863892, 'logps/chosen': -174.48223876953125, 'logps/rejected': -402.43865966796875, 'logps/ref_chosen': -46.00492858886719, 'logps/ref_rejected': -101.88005828857422, 'logits/chosen': -1.6971466541290283, 'logits/rejected': -1.5857012271881104, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.011376879177987576, 'epsilon_dpo/loss_margin_mean': 172.08128356933594, 'epsilon_dpo/beta_margin_mean': 1.9530349969863892, 'epsilon_dpo/beta_margin_std': 1.3995331525802612, 'epsilon_dpo/beta_margin_grad_mean': -0.18481247127056122, 'epsilon_dpo/beta_margin_grad_std': 0.17733608186244965, 'kl/beta': 0.011468926444649696, 'kl/avg_steps': 0.8125, 'epoch': 0.41}
|
|||
|
|
41%|███████████████████████████████████████████████▏ | 282/681 [17:41<16:16, 2.45s/it]
42%|███████████████████████████████████████████████▎ | 283/681 [17:43<16:00, 2.41s/it]
{'loss': 0.3582, 'grad_norm': 47.481971740722656, 'learning_rate': 3.6486760974483685e-07, 'rewards/chosen': -1.330566644668579, 'rewards/rejected': -3.6710352897644043, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.340468645095825, 'logps/chosen': -153.2581329345703, 'logps/rejected': -428.9803466796875, 'logps/ref_chosen': -35.3682861328125, 'logps/ref_rejected': -103.27058410644531, 'logits/chosen': -1.6741735935211182, 'logits/rejected': -1.502656102180481, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.01127807516604662, 'epsilon_dpo/loss_margin_mean': 207.81991577148438, 'epsilon_dpo/beta_margin_mean': 2.340468645095825, 'epsilon_dpo/beta_margin_std': 1.3866698741912842, 'epsilon_dpo/beta_margin_grad_mean': -0.1464729756116867, 'epsilon_dpo/beta_margin_grad_std': 0.15092232823371887, 'kl/beta': 0.011376491747796535, 'kl/avg_steps': 0.875, 'epoch': 0.42}
|
|||
|
|
42%|███████████████████████████████████████████████▎ | 283/681 [17:43<16:00, 2.41s/it]
42%|███████████████████████████████████████████████▌ | 284/681 [17:46<16:11, 2.45s/it]
{'loss': 0.4919, 'grad_norm': 64.37258911132812, 'learning_rate': 3.6372625621898863e-07, 'rewards/chosen': -1.4272245168685913, 'rewards/rejected': -3.583281993865967, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.156057357788086, 'logps/chosen': -168.32333374023438, 'logps/rejected': -419.96551513671875, 'logps/ref_chosen': -41.10857391357422, 'logps/ref_rejected': -99.55363464355469, 'logits/chosen': -1.67313551902771, 'logits/rejected': -1.5210282802581787, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.011197871528565884, 'epsilon_dpo/loss_margin_mean': 193.1970977783203, 'epsilon_dpo/beta_margin_mean': 2.156057596206665, 'epsilon_dpo/beta_margin_std': 1.6328465938568115, 'epsilon_dpo/beta_margin_grad_mean': -0.18323473632335663, 'epsilon_dpo/beta_margin_grad_std': 0.1965819150209427, 'kl/beta': 0.011277811601758003, 'kl/avg_steps': 0.71875, 'epoch': 0.42}
|
|||
|
|
42%|███████████████████████████████████████████████▌ | 284/681 [17:46<16:11, 2.45s/it]
42%|███████████████████████████████████████████████▋ | 285/681 [17:49<16:28, 2.50s/it]
{'loss': 0.4048, 'grad_norm': 60.48746871948242, 'learning_rate': 3.625819059005228e-07, 'rewards/chosen': -1.4201911687850952, 'rewards/rejected': -3.7171897888183594, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.2969985008239746, 'logps/chosen': -163.57217407226562, 'logps/rejected': -443.7559814453125, 'logps/ref_chosen': -35.757301330566406, 'logps/ref_rejected': -108.66427612304688, 'logits/chosen': -1.64085054397583, 'logits/rejected': -1.4849854707717896, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.011096964590251446, 'epsilon_dpo/loss_margin_mean': 207.27685546875, 'epsilon_dpo/beta_margin_mean': 2.2969985008239746, 'epsilon_dpo/beta_margin_std': 1.3974249362945557, 'epsilon_dpo/beta_margin_grad_mean': -0.14801117777824402, 'epsilon_dpo/beta_margin_grad_std': 0.17136132717132568, 'kl/beta': 0.011197330430150032, 'kl/avg_steps': 0.90625, 'epoch': 0.42}
|
|||
|
|
42%|███████████████████████████████████████████████▋ | 285/681 [17:49<16:28, 2.50s/it]
42%|███████████████████████████████████████████████▉ | 286/681 [17:52<17:27, 2.65s/it]
{'loss': 0.4342, 'grad_norm': 60.148529052734375, 'learning_rate': 3.614345889441346e-07, 'rewards/chosen': -1.4335534572601318, 'rewards/rejected': -3.6558303833007812, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.2222766876220703, 'logps/chosen': -175.14715576171875, 'logps/rejected': -428.2286376953125, 'logps/ref_chosen': -45.06391525268555, 'logps/ref_rejected': -95.78263854980469, 'logits/chosen': -1.7381117343902588, 'logits/rejected': -1.4690245389938354, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.011004237458109856, 'epsilon_dpo/loss_margin_mean': 202.3627471923828, 'epsilon_dpo/beta_margin_mean': 2.2222766876220703, 'epsilon_dpo/beta_margin_std': 1.464618444442749, 'epsilon_dpo/beta_margin_grad_mean': -0.16391685605049133, 'epsilon_dpo/beta_margin_grad_std': 0.18253783881664276, 'kl/beta': 0.011096766218543053, 'kl/avg_steps': 0.84375, 'epoch': 0.42}
|
|||
|
|
42%|███████████████████████████████████████████████▉ | 286/681 [17:52<17:27, 2.65s/it]
42%|████████████████████████████████████████████████ | 287/681 [17:54<17:01, 2.59s/it]
{'loss': 0.611, 'grad_norm': 71.68363189697266, 'learning_rate': 3.6028433558269275e-07, 'rewards/chosen': -1.551443099975586, 'rewards/rejected': -3.22538423538208, 'rewards/accuracies': 0.84375, 'rewards/margins': 1.6739410161972046, 'logps/chosen': -186.29312133789062, 'logps/rejected': -378.27142333984375, 'logps/ref_chosen': -44.68206787109375, 'logps/ref_rejected': -82.90010070800781, 'logits/chosen': -1.7347445487976074, 'logits/rejected': -1.4353196620941162, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.010929361917078495, 'epsilon_dpo/loss_margin_mean': 153.76026916503906, 'epsilon_dpo/beta_margin_mean': 1.6739410161972046, 'epsilon_dpo/beta_margin_std': 1.4721882343292236, 'epsilon_dpo/beta_margin_grad_mean': -0.23106051981449127, 'epsilon_dpo/beta_margin_grad_std': 0.19667471945285797, 'kl/beta': 0.011003920808434486, 'kl/avg_steps': 0.6875, 'epoch': 0.42}
|
|||
|
|
42%|████████████████████████████████████████████████ | 287/681 [17:54<17:01, 2.59s/it]
42%|████████████████████████████████████████████████▏ | 288/681 [17:57<16:52, 2.58s/it]
{'loss': 0.3983, 'grad_norm': 56.249759674072266, 'learning_rate': 3.5913117612644327e-07, 'rewards/chosen': -1.3546841144561768, 'rewards/rejected': -3.5305793285369873, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.1758952140808105, 'logps/chosen': -160.89662170410156, 'logps/rejected': -418.419921875, 'logps/ref_chosen': -35.92053985595703, 'logps/ref_rejected': -92.28993225097656, 'logits/chosen': -1.6811950206756592, 'logits/rejected': -1.4292771816253662, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.010827410966157913, 'epsilon_dpo/loss_margin_mean': 201.15391540527344, 'epsilon_dpo/beta_margin_mean': 2.1758952140808105, 'epsilon_dpo/beta_margin_std': 1.2892285585403442, 'epsilon_dpo/beta_margin_grad_mean': -0.15691792964935303, 'epsilon_dpo/beta_margin_grad_std': 0.16194118559360504, 'kl/beta': 0.010928785428404808, 'kl/avg_steps': 0.9375, 'epoch': 0.42}
|
|||
|
|
42%|████████████████████████████████████████████████▏ | 288/681 [17:57<16:52, 2.58s/it]
42%|████████████████████████████████████████████████▍ | 289/681 [17:59<16:36, 2.54s/it]
{'loss': 0.5158, 'grad_norm': 53.693756103515625, 'learning_rate': 3.5797514096221024e-07, 'rewards/chosen': -1.222680687904358, 'rewards/rejected': -3.2015156745910645, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.978835105895996, 'logps/chosen': -151.41265869140625, 'logps/rejected': -390.99652099609375, 'logps/ref_chosen': -37.65406036376953, 'logps/ref_rejected': -92.58161163330078, 'logits/chosen': -1.7554616928100586, 'logits/rejected': -1.5511056184768677, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.010733614675700665, 'epsilon_dpo/loss_margin_mean': 184.6563262939453, 'epsilon_dpo/beta_margin_mean': 1.978835105895996, 'epsilon_dpo/beta_margin_std': 1.5322097539901733, 'epsilon_dpo/beta_margin_grad_mean': -0.19883452355861664, 'epsilon_dpo/beta_margin_grad_std': 0.18830937147140503, 'kl/beta': 0.010827279649674892, 'kl/avg_steps': 0.875, 'epoch': 0.42}
|
|||
|
|
42%|████████████████████████████████████████████████▍ | 289/681 [17:59<16:36, 2.54s/it]
43%|████████████████████████████████████████████████▌ | 290/681 [18:01<16:16, 2.50s/it]
{'loss': 0.4168, 'grad_norm': 51.92376708984375, 'learning_rate': 3.568162605525952e-07, 'rewards/chosen': -1.2994060516357422, 'rewards/rejected': -3.3938629627227783, 'rewards/accuracies': 0.953125, 'rewards/margins': 2.094456911087036, 'logps/chosen': -165.3158416748047, 'logps/rejected': -442.55718994140625, 'logps/ref_chosen': -43.256103515625, 'logps/ref_rejected': -123.40228271484375, 'logits/chosen': -1.6529500484466553, 'logits/rejected': -1.7434873580932617, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.01063715573400259, 'epsilon_dpo/loss_margin_mean': 197.09515380859375, 'epsilon_dpo/beta_margin_mean': 2.094456911087036, 'epsilon_dpo/beta_margin_std': 1.3732136487960815, 'epsilon_dpo/beta_margin_grad_mean': -0.17113091051578522, 'epsilon_dpo/beta_margin_grad_std': 0.15478722751140594, 'kl/beta': 0.010733362287282944, 'kl/avg_steps': 0.90625, 'epoch': 0.43}
|
|||
|
|
43%|████████████████████████████████████████████████▌ | 290/681 [18:01<16:16, 2.50s/it]
43%|████████████████████████████████████████████████▋ | 291/681 [18:04<16:29, 2.54s/it]
{'loss': 0.5064, 'grad_norm': 62.94866943359375, 'learning_rate': 3.5565456543517485e-07, 'rewards/chosen': -1.2878754138946533, 'rewards/rejected': -3.207477569580078, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.9196021556854248, 'logps/chosen': -165.5596466064453, 'logps/rejected': -398.55230712890625, 'logps/ref_chosen': -43.823760986328125, 'logps/ref_rejected': -94.49006652832031, 'logits/chosen': -1.8479794263839722, 'logits/rejected': -1.6821517944335938, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.010556564666330814, 'epsilon_dpo/loss_margin_mean': 182.32635498046875, 'epsilon_dpo/beta_margin_mean': 1.9196021556854248, 'epsilon_dpo/beta_margin_std': 1.422903060913086, 'epsilon_dpo/beta_margin_grad_mean': -0.1979316920042038, 'epsilon_dpo/beta_margin_grad_std': 0.18217770755290985, 'kl/beta': 0.010636964812874794, 'kl/avg_steps': 0.765625, 'epoch': 0.43}
|
|||
|
|
43%|████████████████████████████████████████████████▋ | 291/681 [18:04<16:29, 2.54s/it]
43%|████████████████████████████████████████████████▉ | 292/681 [18:06<16:09, 2.49s/it]
{'loss': 0.4426, 'grad_norm': 62.74113464355469, 'learning_rate': 3.5449008622169583e-07, 'rewards/chosen': -1.1173462867736816, 'rewards/rejected': -3.1657166481018066, 'rewards/accuracies': 0.921875, 'rewards/margins': 2.048370361328125, 'logps/chosen': -141.83103942871094, 'logps/rejected': -397.3466796875, 'logps/ref_chosen': -35.38202667236328, 'logps/ref_rejected': -94.85586547851562, 'logits/chosen': -1.7095630168914795, 'logits/rejected': -1.6350113153457642, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.010471423156559467, 'epsilon_dpo/loss_margin_mean': 196.04180908203125, 'epsilon_dpo/beta_margin_mean': 2.048370122909546, 'epsilon_dpo/beta_margin_std': 1.3248964548110962, 'epsilon_dpo/beta_margin_grad_mean': -0.1748674064874649, 'epsilon_dpo/beta_margin_grad_std': 0.17317946255207062, 'kl/beta': 0.010556144639849663, 'kl/avg_steps': 0.8125, 'epoch': 0.43}
|
|||
|
|
43%|████████████████████████████████████████████████▉ | 292/681 [18:06<16:09, 2.49s/it]
43%|█████████████████████████████████████████████████ | 293/681 [18:09<16:20, 2.53s/it]
{'loss': 0.6168, 'grad_norm': 59.1533088684082, 'learning_rate': 3.5332285359726846e-07, 'rewards/chosen': -1.193084478378296, 'rewards/rejected': -2.81472110748291, 'rewards/accuracies': 0.84375, 'rewards/margins': 1.6216366291046143, 'logps/chosen': -150.8321075439453, 'logps/rejected': -354.53131103515625, 'logps/ref_chosen': -36.39015197753906, 'logps/ref_rejected': -83.55977630615234, 'logits/chosen': -1.8089414834976196, 'logits/rejected': -1.613842248916626, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.010403390973806381, 'epsilon_dpo/loss_margin_mean': 156.5295867919922, 'epsilon_dpo/beta_margin_mean': 1.6216365098953247, 'epsilon_dpo/beta_margin_std': 1.3715225458145142, 'epsilon_dpo/beta_margin_grad_mean': -0.2334311306476593, 'epsilon_dpo/beta_margin_grad_std': 0.19696100056171417, 'kl/beta': 0.010471067391335964, 'kl/avg_steps': 0.65625, 'epoch': 0.43}
|
|||
|
|
43%|█████████████████████████████████████████████████ | 293/681 [18:09<16:20, 2.53s/it]
43%|█████████████████████████████████████████████████▏ | 294/681 [18:11<16:11, 2.51s/it]
{'loss': 0.5324, 'grad_norm': 53.15205001831055, 'learning_rate': 3.5215289831955786e-07, 'rewards/chosen': -1.1122854948043823, 'rewards/rejected': -2.79593563079834, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.6836501359939575, 'logps/chosen': -139.99928283691406, 'logps/rejected': -358.48431396484375, 'logps/ref_chosen': -32.32667541503906, 'logps/ref_rejected': -87.2591552734375, 'logits/chosen': -1.6690821647644043, 'logits/rejected': -1.641129732131958, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.01031605713069439, 'epsilon_dpo/loss_margin_mean': 163.5525360107422, 'epsilon_dpo/beta_margin_mean': 1.6836501359939575, 'epsilon_dpo/beta_margin_std': 1.2057768106460571, 'epsilon_dpo/beta_margin_grad_mean': -0.20596943795681, 'epsilon_dpo/beta_margin_grad_std': 0.16830576956272125, 'kl/beta': 0.010402798652648926, 'kl/avg_steps': 0.84375, 'epoch': 0.43}
|
|||
|
|
43%|█████████████████████████████████████████████████▏ | 294/681 [18:12<16:11, 2.51s/it]
43%|█████████████████████████████████████████████████▍ | 295/681 [18:14<15:59, 2.49s/it]
{'loss': 0.5393, 'grad_norm': 73.17021179199219, 'learning_rate': 3.509802512179737e-07, 'rewards/chosen': -1.1549984216690063, 'rewards/rejected': -3.0700550079345703, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.9150567054748535, 'logps/chosen': -145.589599609375, 'logps/rejected': -396.48590087890625, 'logps/ref_chosen': -32.976951599121094, 'logps/ref_rejected': -96.25013732910156, 'logits/chosen': -1.7157707214355469, 'logits/rejected': -1.6699479818344116, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.010229743085801601, 'epsilon_dpo/loss_margin_mean': 187.62313842773438, 'epsilon_dpo/beta_margin_mean': 1.9150567054748535, 'epsilon_dpo/beta_margin_std': 1.5144377946853638, 'epsilon_dpo/beta_margin_grad_mean': -0.2001977413892746, 'epsilon_dpo/beta_margin_grad_std': 0.19301089644432068, 'kl/beta': 0.010315759107470512, 'kl/avg_steps': 0.84375, 'epoch': 0.43}
|
|||
|
|
43%|█████████████████████████████████████████████████▍ | 295/681 [18:14<15:59, 2.49s/it]
43%|█████████████████████████████████████████████████▌ | 296/681 [18:16<15:48, 2.46s/it]
{'loss': 0.5562, 'grad_norm': 54.07697296142578, 'learning_rate': 3.498049431928577e-07, 'rewards/chosen': -1.1971094608306885, 'rewards/rejected': -2.926664352416992, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.7295548915863037, 'logps/chosen': -159.61997985839844, 'logps/rejected': -388.03179931640625, 'logps/ref_chosen': -41.81062316894531, 'logps/ref_rejected': -99.36541748046875, 'logits/chosen': -1.944000005722046, 'logits/rejected': -1.7023565769195557, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.010147349908947945, 'epsilon_dpo/loss_margin_mean': 170.8570556640625, 'epsilon_dpo/beta_margin_mean': 1.7295547723770142, 'epsilon_dpo/beta_margin_std': 1.38618004322052, 'epsilon_dpo/beta_margin_grad_mean': -0.21665216982364655, 'epsilon_dpo/beta_margin_grad_std': 0.17736758291721344, 'kl/beta': 0.010229448787868023, 'kl/avg_steps': 0.8125, 'epoch': 0.43}
|
|||
|
|
43%|█████████████████████████████████████████████████▌ | 296/681 [18:16<15:48, 2.46s/it]
44%|█████████████████████████████████████████████████▋ | 297/681 [18:19<15:44, 2.46s/it]
{'loss': 0.469, 'grad_norm': 52.08464431762695, 'learning_rate': 3.486270052146694e-07, 'rewards/chosen': -1.112633466720581, 'rewards/rejected': -2.900785446166992, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.7881522178649902, 'logps/chosen': -146.10855102539062, 'logps/rejected': -389.8145751953125, 'logps/ref_chosen': -35.64509582519531, 'logps/ref_rejected': -101.33485412597656, 'logits/chosen': -1.8551688194274902, 'logits/rejected': -1.8467998504638672, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.010062395595014095, 'epsilon_dpo/loss_margin_mean': 178.01626586914062, 'epsilon_dpo/beta_margin_mean': 1.7881520986557007, 'epsilon_dpo/beta_margin_std': 1.1349061727523804, 'epsilon_dpo/beta_margin_grad_mean': -0.19078318774700165, 'epsilon_dpo/beta_margin_grad_std': 0.15408045053482056, 'kl/beta': 0.010147004388272762, 'kl/avg_steps': 0.84375, 'epoch': 0.44}
|
|||
|
|
44%|█████████████████████████████████████████████████▋ | 297/681 [18:19<15:44, 2.46s/it]
44%|█████████████████████████████████████████████████▉ | 298/681 [18:22<16:12, 2.54s/it]
{'loss': 0.4176, 'grad_norm': 47.372406005859375, 'learning_rate': 3.474464683231698e-07, 'rewards/chosen': -0.9769392013549805, 'rewards/rejected': -3.023324966430664, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.0463857650756836, 'logps/chosen': -136.94308471679688, 'logps/rejected': -428.3704833984375, 'logps/ref_chosen': -39.13259506225586, 'logps/ref_rejected': -125.148193359375, 'logits/chosen': -1.854588508605957, 'logits/rejected': -1.974242091178894, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.009971914812922478, 'epsilon_dpo/loss_margin_mean': 205.4117889404297, 'epsilon_dpo/beta_margin_mean': 2.0463857650756836, 'epsilon_dpo/beta_margin_std': 1.3721272945404053, 'epsilon_dpo/beta_margin_grad_mean': -0.17282237112522125, 'epsilon_dpo/beta_margin_grad_std': 0.14743012189865112, 'kl/beta': 0.010062105022370815, 'kl/avg_steps': 0.90625, 'epoch': 0.44}
|
|||
|
|
44%|█████████████████████████████████████████████████▉ | 298/681 [18:22<16:12, 2.54s/it]
44%|██████████████████████████████████████████████████ | 299/681 [18:24<16:27, 2.58s/it]
{'loss': 0.4348, 'grad_norm': 50.078041076660156, 'learning_rate': 3.462633636266041e-07, 'rewards/chosen': -0.933772087097168, 'rewards/rejected': -2.8413944244384766, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.9076223373413086, 'logps/chosen': -122.997314453125, 'logps/rejected': -375.3733215332031, 'logps/ref_chosen': -28.626670837402344, 'logps/ref_rejected': -87.74382781982422, 'logits/chosen': -1.7307016849517822, 'logits/rejected': -1.819953203201294, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.009882355108857155, 'epsilon_dpo/loss_margin_mean': 193.25885009765625, 'epsilon_dpo/beta_margin_mean': 1.9076223373413086, 'epsilon_dpo/beta_margin_std': 1.1600240468978882, 'epsilon_dpo/beta_margin_grad_mean': -0.17700159549713135, 'epsilon_dpo/beta_margin_grad_std': 0.15176741778850555, 'kl/beta': 0.009971735998988152, 'kl/avg_steps': 0.90625, 'epoch': 0.44}
|
|||
|
|
44%|██████████████████████████████████████████████████ | 299/681 [18:24<16:27, 2.58s/it]
44%|██████████████████████████████████████████████████▏ | 300/681 [18:27<16:15, 2.56s/it]
{'loss': 0.4996, 'grad_norm': 58.53467559814453, 'learning_rate': 3.4507772230088147e-07, 'rewards/chosen': -1.0862467288970947, 'rewards/rejected': -3.1269874572753906, 'rewards/accuracies': 0.90625, 'rewards/margins': 2.040740728378296, 'logps/chosen': -144.44647216796875, 'logps/rejected': -423.14324951171875, 'logps/ref_chosen': -33.894203186035156, 'logps/ref_rejected': -103.88007354736328, 'logits/chosen': -1.7490193843841553, 'logits/rejected': -1.7381911277770996, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.009802866727113724, 'epsilon_dpo/loss_margin_mean': 208.71090698242188, 'epsilon_dpo/beta_margin_mean': 2.040740728378296, 'epsilon_dpo/beta_margin_std': 1.4987692832946777, 'epsilon_dpo/beta_margin_grad_mean': -0.1854911744594574, 'epsilon_dpo/beta_margin_grad_std': 0.19396579265594482, 'kl/beta': 0.009882179088890553, 'kl/avg_steps': 0.8125, 'epoch': 0.44}
|
|||
|
|
44%|██████████████████████████████████████████████████▏ | 300/681 [18:27<16:15, 2.56s/it][INFO|trainer.py:4307] 2026-04-18 09:49:49,167 >>
|
|||
|
|
***** Running Evaluation *****
|
|||
|
|
[INFO|trainer.py:4309] 2026-04-18 09:49:49,167 >> Num examples = 2339
|
|||
|
|
[INFO|trainer.py:4312] 2026-04-18 09:49:49,167 >> Batch size = 8
|
|||
|
|
|
|||
|
|
0%| | 0/73 [00:00<?, ?it/s][A
|
|||
|
|
3%|███▏ | 2/73 [00:00<00:18, 3.86it/s][A
|
|||
|
|
4%|████▊ | 3/73 [00:01<00:27, 2.59it/s][A
|
|||
|
|
5%|██████▍ | 4/73 [00:01<00:30, 2.26it/s][A
|
|||
|
|
7%|████████ | 5/73 [00:02<00:32, 2.09it/s][A
|
|||
|
|
8%|█████████▌ | 6/73 [00:02<00:33, 2.02it/s][A
|
|||
|
|
10%|███████████▏ | 7/73 [00:03<00:32, 2.01it/s][A
|
|||
|
|
11%|████████████▊ | 8/73 [00:03<00:31, 2.06it/s][A
|
|||
|
|
12%|██████████████▍ | 9/73 [00:04<00:32, 1.99it/s][A
|
|||
|
|
14%|███████████████▉ | 10/73 [00:04<00:32, 1.95it/s][A
|
|||
|
|
15%|█████████████████▍ | 11/73 [00:05<00:31, 1.97it/s][A
|
|||
|
|
16%|███████████████████ | 12/73 [00:05<00:31, 1.96it/s][A
|
|||
|
|
18%|████████████████████▋ | 13/73 [00:06<00:31, 1.90it/s][A
|
|||
|
|
19%|██████████████████████▏ | 14/73 [00:06<00:30, 1.90it/s][A
|
|||
|
|
21%|███████████████████████▊ | 15/73 [00:07<00:31, 1.87it/s][A
|
|||
|
|
22%|█████████████████████████▍ | 16/73 [00:07<00:30, 1.85it/s][A
|
|||
|
|
23%|███████████████████████████ | 17/73 [00:08<00:29, 1.87it/s][A
|
|||
|
|
25%|████████████████████████████▌ | 18/73 [00:09<00:29, 1.85it/s][A
|
|||
|
|
26%|██████████████████████████████▏ | 19/73 [00:09<00:29, 1.85it/s][A
|
|||
|
|
27%|███████████████████████████████▊ | 20/73 [00:10<00:28, 1.83it/s][A
|
|||
|
|
29%|█████████████████████████████████▎ | 21/73 [00:10<00:28, 1.82it/s][A
|
|||
|
|
30%|██████████████████████████████████▉ | 22/73 [00:11<00:28, 1.82it/s][A
|
|||
|
|
32%|████████████████████████████████████▌ | 23/73 [00:11<00:27, 1.85it/s][A
|
|||
|
|
33%|██████████████████████████████████████▏ | 24/73 [00:12<00:26, 1.88it/s][A
|
|||
|
|
34%|███████████████████████████████████████▋ | 25/73 [00:12<00:25, 1.87it/s][A
|
|||
|
|
36%|█████████████████████████████████████████▎ | 26/73 [00:13<00:24, 1.91it/s][A
|
|||
|
|
37%|██████████████████████████████████████████▉ | 27/73 [00:13<00:22, 2.05it/s][A
|
|||
|
|
38%|████████████████████████████████████████████▍ | 28/73 [00:14<00:22, 2.04it/s][A
|
|||
|
|
40%|██████████████████████████████████████████████ | 29/73 [00:14<00:21, 2.01it/s][A
|
|||
|
|
41%|███████████████████████████████████████████████▋ | 30/73 [00:15<00:21, 1.97it/s][A
|
|||
|
|
42%|█████████████████████████████████████████████████▎ | 31/73 [00:15<00:21, 1.92it/s][A
|
|||
|
|
44%|██████████████████████████████████████████████████▊ | 32/73 [00:16<00:21, 1.94it/s][A
|
|||
|
|
45%|████████████████████████████████████████████████████▍ | 33/73 [00:16<00:20, 1.93it/s][A
|
|||
|
|
47%|██████████████████████████████████████████████████████ | 34/73 [00:17<00:19, 1.97it/s][A
|
|||
|
|
48%|███████████████████████████████████████████████████████▌ | 35/73 [00:17<00:19, 1.90it/s][A
|
|||
|
|
49%|█████████████████████████████████████████████████████████▏ | 36/73 [00:18<00:19, 1.92it/s][A
|
|||
|
|
51%|██████████████████████████████████████████████████████████▊ | 37/73 [00:18<00:19, 1.87it/s][A
|
|||
|
|
52%|████████████████████████████████████████████████████████████▍ | 38/73 [00:19<00:18, 1.93it/s][A
|
|||
|
|
53%|█████████████████████████████████████████████████████████████▉ | 39/73 [00:19<00:17, 1.92it/s][A
|
|||
|
|
55%|███████████████████████████████████████████████████████████████▌ | 40/73 [00:20<00:17, 1.93it/s][A
|
|||
|
|
56%|█████████████████████████████████████████████████████████████████▏ | 41/73 [00:20<00:16, 1.94it/s][A
|
|||
|
|
58%|██████████████████████████████████████████████████████████████████▋ | 42/73 [00:21<00:16, 1.91it/s][A
|
|||
|
|
59%|████████████████████████████████████████████████████████████████████▎ | 43/73 [00:22<00:15, 1.88it/s][A
|
|||
|
|
60%|█████████████████████████████████████████████████████████████████████▉ | 44/73 [00:22<00:15, 1.86it/s][A
|
|||
|
|
62%|███████████████████████████████████████████████████████████████████████▌ | 45/73 [00:23<00:15, 1.83it/s][A
|
|||
|
|
63%|█████████████████████████████████████████████████████████████████████████ | 46/73 [00:23<00:14, 1.87it/s][A
|
|||
|
|
64%|██████████████████████████████████████████████████████████████████████████▋ | 47/73 [00:24<00:13, 1.89it/s][A
|
|||
|
|
66%|████████████████████████████████████████████████████████████████████████████▎ | 48/73 [00:24<00:12, 1.93it/s][A
|
|||
|
|
67%|█████████████████████████████████████████████████████████████████████████████▊ | 49/73 [00:25<00:12, 1.88it/s][A
|
|||
|
|
68%|███████████████████████████████████████████████████████████████████████████████▍ | 50/73 [00:25<00:12, 1.86it/s][A
|
|||
|
|
70%|█████████████████████████████████████████████████████████████████████████████████ | 51/73 [00:26<00:11, 1.84it/s][A
|
|||
|
|
71%|██████████████████████████████████████████████████████████████████████████████████▋ | 52/73 [00:26<00:11, 1.83it/s][A
|
|||
|
|
73%|████████████████████████████████████████████████████████████████████████████████████▏ | 53/73 [00:27<00:10, 1.94it/s][A
|
|||
|
|
74%|█████████████████████████████████████████████████████████████████████████████████████▊ | 54/73 [00:27<00:09, 1.97it/s][A
|
|||
|
|
75%|███████████████████████████████████████████████████████████████████████████████████████▍ | 55/73 [00:28<00:08, 2.01it/s][A
|
|||
|
|
77%|████████████████████████████████████████████████████████████████████████████████████████▉ | 56/73 [00:28<00:08, 1.96it/s][A
|
|||
|
|
78%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 57/73 [00:29<00:08, 1.91it/s][A
|
|||
|
|
79%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 58/73 [00:29<00:07, 1.94it/s][A
|
|||
|
|
81%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 59/73 [00:30<00:07, 1.88it/s][A
|
|||
|
|
82%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 60/73 [00:31<00:06, 1.88it/s][A
|
|||
|
|
84%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 61/73 [00:31<00:06, 1.87it/s][A
|
|||
|
|
85%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 62/73 [00:32<00:05, 1.84it/s][A
|
|||
|
|
86%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 63/73 [00:32<00:05, 1.90it/s][A
|
|||
|
|
88%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 64/73 [00:33<00:04, 1.92it/s][A
|
|||
|
|
89%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 65/73 [00:33<00:04, 1.87it/s][A
|
|||
|
|
90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 66/73 [00:34<00:03, 1.86it/s][A
|
|||
|
|
92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 67/73 [00:34<00:03, 1.84it/s][A
|
|||
|
|
93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 68/73 [00:35<00:02, 1.86it/s][A
|
|||
|
|
95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 69/73 [00:35<00:02, 1.98it/s][A
|
|||
|
|
96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 70/73 [00:36<00:01, 1.92it/s][A
|
|||
|
|
97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 71/73 [00:36<00:01, 1.90it/s][A
|
|||
|
|
99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 72/73 [00:37<00:00, 1.89it/s][A
|
|||
|
|
100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 73/73 [00:37<00:00, 2.00it/s][A
|
|||
|
|
[A{'eval_loss': 0.41313475370407104, 'eval_runtime': 38.3815, 'eval_samples_per_second': 60.941, 'eval_steps_per_second': 1.928, 'eval_epsilon_dpo/beta': 0.009748955257236958, 'eval_epsilon_dpo/loss_margin_mean': 122.25267791748047, 'eval_epsilon_dpo/beta_margin_mean': 1.1843825578689575, 'eval_epsilon_dpo/beta_margin_std': 1.3051005601882935, 'eval_epsilon_dpo/beta_margin_grad_mean': -0.29610493779182434, 'eval_epsilon_dpo/beta_margin_grad_std': 0.20769952237606049, 'eval_rewards/chosen': -1.5382959842681885, 'eval_rewards/rejected': -2.7226786613464355, 'eval_rewards/accuracies': 0.7928082346916199, 'eval_rewards/margins': 1.1843825578689575, 'eval_logps/chosen': -225.67529296875, 'eval_logps/rejected': -371.7172546386719, 'eval_logps/ref_chosen': -68.29110717773438, 'eval_logps/ref_rejected': -92.08038330078125, 'eval_logits/chosen': -1.9427273273468018, 'eval_logits/rejected': -1.7790117263793945, 'eval_kl/p_epsilon_steps': 0.778253436088562, 'eval_kl/n_epsilon_steps': 0.22174657881259918, 'epoch': 0.44}
|
|||
|
|
44%|██████████████████████████████████████████████████▏ | 300/681 [19:05<16:15, 2.56s/it]
|
|||
|
|
100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 73/73 [00:37<00:00, 2.00it/s][A
|
|||
|
|
[A
44%|█████████████████████████████████████████████████▌ | 301/681 [19:07<1:28:38, 14.00s/it]
{'loss': 0.402, 'grad_norm': 49.01324462890625, 'learning_rate': 3.4388957558875316e-07, 'rewards/chosen': -1.0266605615615845, 'rewards/rejected': -3.002654790878296, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.9759942293167114, 'logps/chosen': -140.640380859375, 'logps/rejected': -407.7601013183594, 'logps/ref_chosen': -35.052242279052734, 'logps/ref_rejected': -98.54647827148438, 'logits/chosen': -1.7920889854431152, 'logits/rejected': -1.7111059427261353, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.009714669547975063, 'epsilon_dpo/loss_margin_mean': 203.62547302246094, 'epsilon_dpo/beta_margin_mean': 1.9759942293167114, 'epsilon_dpo/beta_margin_std': 1.1226637363433838, 'epsilon_dpo/beta_margin_grad_mean': -0.16645051538944244, 'epsilon_dpo/beta_margin_grad_std': 0.14359311759471893, 'kl/beta': 0.009802533313632011, 'kl/avg_steps': 0.90625, 'epoch': 0.44}
|
|||
|
|
44%|█████████████████████████████████████████████████▌ | 301/681 [19:07<1:28:38, 14.00s/it]
44%|█████████████████████████████████████████████████▋ | 302/681 [19:10<1:06:37, 10.55s/it]
{'loss': 0.4794, 'grad_norm': 63.02902603149414, 'learning_rate': 3.426989547989902e-07, 'rewards/chosen': -0.9009679555892944, 'rewards/rejected': -2.8061742782592773, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.9052062034606934, 'logps/chosen': -124.57978820800781, 'logps/rejected': -395.2374267578125, 'logps/ref_chosen': -31.218624114990234, 'logps/ref_rejected': -103.73237609863281, 'logits/chosen': -1.8670696020126343, 'logits/rejected': -1.852245807647705, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.009633492678403854, 'epsilon_dpo/loss_margin_mean': 198.14389038085938, 'epsilon_dpo/beta_margin_mean': 1.9052062034606934, 'epsilon_dpo/beta_margin_std': 1.2975436449050903, 'epsilon_dpo/beta_margin_grad_mean': -0.1874367892742157, 'epsilon_dpo/beta_margin_grad_std': 0.1728639304637909, 'kl/beta': 0.00971449539065361, 'kl/avg_steps': 0.84375, 'epoch': 0.44}
|
|||
|
|
44%|█████████████████████████████████████████████████▋ | 302/681 [19:10<1:06:37, 10.55s/it]
44%|██████████████████████████████████████████████████▋ | 303/681 [19:12<50:54, 8.08s/it]
{'loss': 0.5374, 'grad_norm': 64.47979736328125, 'learning_rate': 3.4150589130555773e-07, 'rewards/chosen': -0.9886916875839233, 'rewards/rejected': -2.8725757598876953, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.8838841915130615, 'logps/chosen': -145.26632690429688, 'logps/rejected': -393.965087890625, 'logps/ref_chosen': -42.143306732177734, 'logps/ref_rejected': -93.17671203613281, 'logits/chosen': -1.9428577423095703, 'logits/rejected': -1.761818289756775, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.009558911435306072, 'epsilon_dpo/loss_margin_mean': 197.6653594970703, 'epsilon_dpo/beta_margin_mean': 1.8838841915130615, 'epsilon_dpo/beta_margin_std': 1.4101883172988892, 'epsilon_dpo/beta_margin_grad_mean': -0.1947356015443802, 'epsilon_dpo/beta_margin_grad_std': 0.19330711662769318, 'kl/beta': 0.009633215144276619, 'kl/avg_steps': 0.78125, 'epoch': 0.44}
|
|||
|
|
44%|██████████████████████████████████████████████████▋ | 303/681 [19:12<50:54, 8.08s/it]
45%|██████████████████████████████████████████████████▉ | 304/681 [19:15<40:18, 6.41s/it]
{'loss': 0.4258, 'grad_norm': 50.6738395690918, 'learning_rate': 3.403104165467883e-07, 'rewards/chosen': -0.9677666425704956, 'rewards/rejected': -2.951878070831299, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.9841113090515137, 'logps/chosen': -144.84146118164062, 'logps/rejected': -408.85595703125, 'logps/ref_chosen': -42.97766876220703, 'logps/ref_rejected': -97.31230926513672, 'logits/chosen': -1.9802241325378418, 'logits/rejected': -1.827461838722229, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.009478837251663208, 'epsilon_dpo/loss_margin_mean': 209.67984008789062, 'epsilon_dpo/beta_margin_mean': 1.9841113090515137, 'epsilon_dpo/beta_margin_std': 1.1967487335205078, 'epsilon_dpo/beta_margin_grad_mean': -0.17279693484306335, 'epsilon_dpo/beta_margin_grad_std': 0.15856152772903442, 'kl/beta': 0.0095585398375988, 'kl/avg_steps': 0.84375, 'epoch': 0.45}
|
|||
|
|
45%|██████████████████████████████████████████████████▉ | 304/681 [19:15<40:18, 6.41s/it]
45%|███████████████████████████████████████████████████ | 305/681 [19:17<32:48, 5.24s/it]
{'loss': 0.4923, 'grad_norm': 49.391204833984375, 'learning_rate': 3.391125620245535e-07, 'rewards/chosen': -1.0703450441360474, 'rewards/rejected': -2.8853065967559814, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.8149614334106445, 'logps/chosen': -159.82958984375, 'logps/rejected': -405.56689453125, 'logps/ref_chosen': -46.15346145629883, 'logps/ref_rejected': -98.42967224121094, 'logits/chosen': -1.9485129117965698, 'logits/rejected': -1.8468644618988037, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.009399528615176678, 'epsilon_dpo/loss_margin_mean': 193.46107482910156, 'epsilon_dpo/beta_margin_mean': 1.814961552619934, 'epsilon_dpo/beta_margin_std': 1.2615888118743896, 'epsilon_dpo/beta_margin_grad_mean': -0.1955670863389969, 'epsilon_dpo/beta_margin_grad_std': 0.16656768321990967, 'kl/beta': 0.009478564374148846, 'kl/avg_steps': 0.84375, 'epoch': 0.45}
|
|||
|
|
45%|███████████████████████████████████████████████████ | 305/681 [19:17<32:48, 5.24s/it]
45%|███████████████████████████████████████████████████▏ | 306/681 [19:20<27:46, 4.44s/it]
{'loss': 0.4142, 'grad_norm': 44.052284240722656, 'learning_rate': 3.3791235930343417e-07, 'rewards/chosen': -1.0861380100250244, 'rewards/rejected': -2.952488899230957, 'rewards/accuracies': 0.984375, 'rewards/margins': 1.8663511276245117, 'logps/chosen': -163.22915649414062, 'logps/rejected': -408.5956115722656, 'logps/ref_chosen': -46.639854431152344, 'logps/ref_rejected': -91.45211791992188, 'logits/chosen': -1.9102896451950073, 'logits/rejected': -1.7865833044052124, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.009312070906162262, 'epsilon_dpo/loss_margin_mean': 200.55418395996094, 'epsilon_dpo/beta_margin_mean': 1.8663511276245117, 'epsilon_dpo/beta_margin_std': 1.0456689596176147, 'epsilon_dpo/beta_margin_grad_mean': -0.17502400279045105, 'epsilon_dpo/beta_margin_grad_std': 0.13139215111732483, 'kl/beta': 0.009399257600307465, 'kl/avg_steps': 0.9375, 'epoch': 0.45}
|
|||
|
|
45%|███████████████████████████████████████████████████▏ | 306/681 [19:20<27:46, 4.44s/it]
45%|███████████████████████████████████████████████████▍ | 307/681 [19:22<24:09, 3.88s/it]
{'loss': 0.5766, 'grad_norm': 60.684417724609375, 'learning_rate': 3.367098400098881e-07, 'rewards/chosen': -0.9867569208145142, 'rewards/rejected': -2.6786000728607178, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.6918432712554932, 'logps/chosen': -150.8052978515625, 'logps/rejected': -382.8536376953125, 'logps/ref_chosen': -44.40485382080078, 'logps/ref_rejected': -92.84294128417969, 'logits/chosen': -1.928983449935913, 'logits/rejected': -1.8208749294281006, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.009248863905668259, 'epsilon_dpo/loss_margin_mean': 183.6102294921875, 'epsilon_dpo/beta_margin_mean': 1.6918431520462036, 'epsilon_dpo/beta_margin_std': 1.3128682374954224, 'epsilon_dpo/beta_margin_grad_mean': -0.2182687371969223, 'epsilon_dpo/beta_margin_grad_std': 0.1931067556142807, 'kl/beta': 0.009311958216130733, 'kl/avg_steps': 0.6875, 'epoch': 0.45}
|
|||
|
|
45%|███████████████████████████████████████████████████▍ | 307/681 [19:22<24:09, 3.88s/it]
45%|███████████████████████████████████████████████████▌ | 308/681 [19:25<21:11, 3.41s/it]
{'loss': 0.5679, 'grad_norm': 64.62168884277344, 'learning_rate': 3.355050358314172e-07, 'rewards/chosen': -1.161744236946106, 'rewards/rejected': -2.8002285957336426, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.638484239578247, 'logps/chosen': -164.06167602539062, 'logps/rejected': -391.16510009765625, 'logps/ref_chosen': -37.731178283691406, 'logps/ref_rejected': -85.77114868164062, 'logits/chosen': -1.9160457849502563, 'logits/rejected': -1.7217934131622314, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.009177041240036488, 'epsilon_dpo/loss_margin_mean': 179.0634307861328, 'epsilon_dpo/beta_margin_mean': 1.6384843587875366, 'epsilon_dpo/beta_margin_std': 1.205591082572937, 'epsilon_dpo/beta_margin_grad_mean': -0.21642521023750305, 'epsilon_dpo/beta_margin_grad_std': 0.18705737590789795, 'kl/beta': 0.00924837589263916, 'kl/avg_steps': 0.78125, 'epoch': 0.45}
|
|||
|
|
45%|███████████████████████████████████████████████████▌ | 308/681 [19:25<21:11, 3.41s/it]
45%|███████████████████████████████████████████████████▋ | 309/681 [19:27<19:20, 3.12s/it]
{'loss': 0.5253, 'grad_norm': 56.110897064208984, 'learning_rate': 3.3429797851573183e-07, 'rewards/chosen': -0.9513264894485474, 'rewards/rejected': -2.713845729827881, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.7625192403793335, 'logps/chosen': -139.52688598632812, 'logps/rejected': -384.9780578613281, 'logps/ref_chosen': -35.310665130615234, 'logps/ref_rejected': -86.65730285644531, 'logits/chosen': -1.9111260175704956, 'logits/rejected': -1.7678544521331787, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.009103032760322094, 'epsilon_dpo/loss_margin_mean': 194.1045379638672, 'epsilon_dpo/beta_margin_mean': 1.7625192403793335, 'epsilon_dpo/beta_margin_std': 1.2436327934265137, 'epsilon_dpo/beta_margin_grad_mean': -0.20181921124458313, 'epsilon_dpo/beta_margin_grad_std': 0.18235483765602112, 'kl/beta': 0.00917668268084526, 'kl/avg_steps': 0.8125, 'epoch': 0.45}
|
|||
|
|
45%|███████████████████████████████████████████████████▋ | 309/681 [19:27<19:20, 3.12s/it]
46%|███████████████████████████████████████████████████▉ | 310/681 [19:30<18:33, 3.00s/it]
{'loss': 0.5391, 'grad_norm': 57.408546447753906, 'learning_rate': 3.3308869986991487e-07, 'rewards/chosen': -1.0019582509994507, 'rewards/rejected': -2.7351129055023193, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.7331546545028687, 'logps/chosen': -152.9647674560547, 'logps/rejected': -388.26837158203125, 'logps/ref_chosen': -42.4453125, 'logps/ref_rejected': -85.33100128173828, 'logits/chosen': -1.9054917097091675, 'logits/rejected': -1.831470012664795, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.009035356342792511, 'epsilon_dpo/loss_margin_mean': 192.4178924560547, 'epsilon_dpo/beta_margin_mean': 1.7331546545028687, 'epsilon_dpo/beta_margin_std': 1.237532138824463, 'epsilon_dpo/beta_margin_grad_mean': -0.2072700560092926, 'epsilon_dpo/beta_margin_grad_std': 0.18662025034427643, 'kl/beta': 0.009102723561227322, 'kl/avg_steps': 0.75, 'epoch': 0.46}
|
|||
|
|
46%|███████████████████████████████████████████████████▉ | 310/681 [19:30<18:33, 3.00s/it]
46%|████████████████████████████████████████████████████ | 311/681 [19:33<18:00, 2.92s/it]
{'loss': 0.3892, 'grad_norm': 37.80696487426758, 'learning_rate': 3.3187723175958346e-07, 'rewards/chosen': -0.9435994029045105, 'rewards/rejected': -3.0694334506988525, 'rewards/accuracies': 0.96875, 'rewards/margins': 2.1258339881896973, 'logps/chosen': -135.50538635253906, 'logps/rejected': -425.759765625, 'logps/ref_chosen': -30.15520477294922, 'logps/ref_rejected': -82.74742126464844, 'logits/chosen': -1.8577749729156494, 'logits/rejected': -1.6029454469680786, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.008951152674853802, 'epsilon_dpo/loss_margin_mean': 237.6621856689453, 'epsilon_dpo/beta_margin_mean': 2.1258339881896973, 'epsilon_dpo/beta_margin_std': 1.3122553825378418, 'epsilon_dpo/beta_margin_grad_mean': -0.1625707447528839, 'epsilon_dpo/beta_margin_grad_std': 0.14155903458595276, 'kl/beta': 0.0090349605306983, 'kl/avg_steps': 0.9375, 'epoch': 0.46}
|
|||
|
|
46%|████████████████████████████████████████████████████ | 311/681 [19:33<18:00, 2.92s/it]
46%|████████████████████████████████████████████████████▏ | 312/681 [19:35<16:40, 2.71s/it]
{'loss': 0.5417, 'grad_norm': 56.1854362487793, 'learning_rate': 3.306636061080487e-07, 'rewards/chosen': -1.088057041168213, 'rewards/rejected': -2.8246212005615234, 'rewards/accuracies': 0.875, 'rewards/margins': 1.7365641593933105, 'logps/chosen': -154.71817016601562, 'logps/rejected': -399.96337890625, 'logps/ref_chosen': -32.57682418823242, 'logps/ref_rejected': -81.77426147460938, 'logits/chosen': -1.8750476837158203, 'logits/rejected': -1.6363931894302368, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.008882002905011177, 'epsilon_dpo/loss_margin_mean': 196.04774475097656, 'epsilon_dpo/beta_margin_mean': 1.7365641593933105, 'epsilon_dpo/beta_margin_std': 1.3034473657608032, 'epsilon_dpo/beta_margin_grad_mean': -0.209417924284935, 'epsilon_dpo/beta_margin_grad_std': 0.18264873325824738, 'kl/beta': 0.008951044641435146, 'kl/avg_steps': 0.78125, 'epoch': 0.46}
|
|||
|
|
46%|████████████████████████████████████████████████████▏ | 312/681 [19:35<16:40, 2.71s/it]
46%|████████████████████████████████████████████████████▍ | 313/681 [19:37<16:03, 2.62s/it]
{'loss': 0.4862, 'grad_norm': 47.559181213378906, 'learning_rate': 3.2944785489547537e-07, 'rewards/chosen': -1.0461211204528809, 'rewards/rejected': -2.8530638217926025, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.8069427013397217, 'logps/chosen': -147.1026153564453, 'logps/rejected': -417.8107604980469, 'logps/ref_chosen': -28.45586395263672, 'logps/ref_rejected': -93.60973358154297, 'logits/chosen': -1.9602954387664795, 'logits/rejected': -1.78814697265625, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.008807598613202572, 'epsilon_dpo/loss_margin_mean': 205.55426025390625, 'epsilon_dpo/beta_margin_mean': 1.8069427013397217, 'epsilon_dpo/beta_margin_std': 1.2012414932250977, 'epsilon_dpo/beta_margin_grad_mean': -0.19396935403347015, 'epsilon_dpo/beta_margin_grad_std': 0.16426628828048706, 'kl/beta': 0.008881657384335995, 'kl/avg_steps': 0.84375, 'epoch': 0.46}
|
|||
|
|
46%|████████████████████████████████████████████████████▍ | 313/681 [19:37<16:03, 2.62s/it]
46%|████████████████████████████████████████████████████▌ | 314/681 [19:40<15:39, 2.56s/it]
{'loss': 0.4832, 'grad_norm': 43.1519660949707, 'learning_rate': 3.2823001015803857e-07, 'rewards/chosen': -0.953238308429718, 'rewards/rejected': -2.694100856781006, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.7408623695373535, 'logps/chosen': -144.1108856201172, 'logps/rejected': -411.68084716796875, 'logps/ref_chosen': -34.89348220825195, 'logps/ref_rejected': -102.7928695678711, 'logits/chosen': -1.9734838008880615, 'logits/rejected': -1.930768370628357, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.008725648745894432, 'epsilon_dpo/loss_margin_mean': 199.67056274414062, 'epsilon_dpo/beta_margin_mean': 1.7408623695373535, 'epsilon_dpo/beta_margin_std': 1.1878901720046997, 'epsilon_dpo/beta_margin_grad_mean': -0.19880293309688568, 'epsilon_dpo/beta_margin_grad_std': 0.14619764685630798, 'kl/beta': 0.008807345293462276, 'kl/avg_steps': 0.9375, 'epoch': 0.46}
|
|||
|
|
46%|████████████████████████████████████████████████████▌ | 314/681 [19:40<15:39, 2.56s/it]
46%|████████████████████████████████████████████████████▋ | 315/681 [19:42<15:33, 2.55s/it]
{'loss': 0.5973, 'grad_norm': 62.91958999633789, 'learning_rate': 3.270101039870797e-07, 'rewards/chosen': -1.0994541645050049, 'rewards/rejected': -2.584402084350586, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.4849481582641602, 'logps/chosen': -159.944091796875, 'logps/rejected': -384.639404296875, 'logps/ref_chosen': -33.06333923339844, 'logps/ref_rejected': -85.8106918334961, 'logits/chosen': -1.8116509914398193, 'logits/rejected': -1.7590763568878174, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.008652785792946815, 'epsilon_dpo/loss_margin_mean': 171.94798278808594, 'epsilon_dpo/beta_margin_mean': 1.4849480390548706, 'epsilon_dpo/beta_margin_std': 1.1557073593139648, 'epsilon_dpo/beta_margin_grad_mean': -0.23646311461925507, 'epsilon_dpo/beta_margin_grad_std': 0.16848552227020264, 'kl/beta': 0.008725542575120926, 'kl/avg_steps': 0.84375, 'epoch': 0.46}
|
|||
|
|
46%|████████████████████████████████████████████████████▋ | 315/681 [19:42<15:33, 2.55s/it]
46%|████████████████████████████████████████████████████▉ | 316/681 [19:45<15:12, 2.50s/it]
{'loss': 0.4775, 'grad_norm': 53.759849548339844, 'learning_rate': 3.2578816852826086e-07, 'rewards/chosen': -1.1298701763153076, 'rewards/rejected': -2.8957996368408203, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.7659293413162231, 'logps/chosen': -171.6690216064453, 'logps/rejected': -445.84991455078125, 'logps/ref_chosen': -39.93564224243164, 'logps/ref_rejected': -107.95684814453125, 'logits/chosen': -1.918544054031372, 'logits/rejected': -1.8977160453796387, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.008572276681661606, 'epsilon_dpo/loss_margin_mean': 206.15966796875, 'epsilon_dpo/beta_margin_mean': 1.7659293413162231, 'epsilon_dpo/beta_margin_std': 1.222463607788086, 'epsilon_dpo/beta_margin_grad_mean': -0.19783826172351837, 'epsilon_dpo/beta_margin_grad_std': 0.14127233624458313, 'kl/beta': 0.008652537129819393, 'kl/avg_steps': 0.9375, 'epoch': 0.46}
|
|||
|
|
46%|████████████████████████████████████████████████████▉ | 316/681 [19:45<15:12, 2.50s/it]
47%|█████████████████████████████████████████████████████ | 317/681 [19:47<15:22, 2.53s/it]
{'loss': 0.4319, 'grad_norm': 45.539825439453125, 'learning_rate': 3.2456423598071783e-07, 'rewards/chosen': -1.0545722246170044, 'rewards/rejected': -3.0235424041748047, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.9689700603485107, 'logps/chosen': -157.90228271484375, 'logps/rejected': -463.1947021484375, 'logps/ref_chosen': -33.96290969848633, 'logps/ref_rejected': -107.23854064941406, 'logits/chosen': -1.982346773147583, 'logits/rejected': -1.7732250690460205, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.00850069522857666, 'epsilon_dpo/loss_margin_mean': 232.01678466796875, 'epsilon_dpo/beta_margin_mean': 1.9689700603485107, 'epsilon_dpo/beta_margin_std': 1.239305853843689, 'epsilon_dpo/beta_margin_grad_mean': -0.1773056834936142, 'epsilon_dpo/beta_margin_grad_std': 0.1531369686126709, 'kl/beta': 0.008572173304855824, 'kl/avg_steps': 0.84375, 'epoch': 0.47}
|
|||
|
|
47%|█████████████████████████████████████████████████████ | 317/681 [19:47<15:22, 2.53s/it]
47%|█████████████████████████████████████████████████████▏ | 318/681 [19:50<14:58, 2.48s/it]
{'loss': 0.5493, 'grad_norm': 55.2903938293457, 'learning_rate': 3.233383385962115e-07, 'rewards/chosen': -1.177966833114624, 'rewards/rejected': -2.810570240020752, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.6326031684875488, 'logps/chosen': -180.26669311523438, 'logps/rejected': -422.9417419433594, 'logps/ref_chosen': -40.75099563598633, 'logps/ref_rejected': -89.30494689941406, 'logits/chosen': -2.0958797931671143, 'logits/rejected': -1.8145737648010254, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.00842957105487585, 'epsilon_dpo/loss_margin_mean': 194.12109375, 'epsilon_dpo/beta_margin_mean': 1.6326032876968384, 'epsilon_dpo/beta_margin_std': 1.1618354320526123, 'epsilon_dpo/beta_margin_grad_mean': -0.21183888614177704, 'epsilon_dpo/beta_margin_grad_std': 0.17660567164421082, 'kl/beta': 0.008500450290739536, 'kl/avg_steps': 0.84375, 'epoch': 0.47}
|
|||
|
|
47%|█████████████████████████████████████████████████████▏ | 318/681 [19:50<14:58, 2.48s/it]
47%|█████████████████████████████████████████████████████▍ | 319/681 [19:52<14:57, 2.48s/it]
{'loss': 0.4951, 'grad_norm': 45.54665756225586, 'learning_rate': 3.2211050867827805e-07, 'rewards/chosen': -0.8743832111358643, 'rewards/rejected': -2.826244354248047, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.9518612623214722, 'logps/chosen': -135.60565185546875, 'logps/rejected': -459.826416015625, 'logps/ref_chosen': -31.367862701416016, 'logps/ref_rejected': -121.58282470703125, 'logits/chosen': -2.003638982772827, 'logits/rejected': -2.138455867767334, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.008361675776541233, 'epsilon_dpo/loss_margin_mean': 234.00582885742188, 'epsilon_dpo/beta_margin_mean': 1.9518612623214722, 'epsilon_dpo/beta_margin_std': 1.3541351556777954, 'epsilon_dpo/beta_margin_grad_mean': -0.18624837696552277, 'epsilon_dpo/beta_margin_grad_std': 0.18912473320960999, 'kl/beta': 0.008429327979683876, 'kl/avg_steps': 0.8125, 'epoch': 0.47}
|
|||
|
|
47%|█████████████████████████████████████████████████████▍ | 319/681 [19:52<14:57, 2.48s/it]
47%|█████████████████████████████████████████████████████▌ | 320/681 [19:54<14:56, 2.48s/it]
{'loss': 0.4546, 'grad_norm': 42.13555908203125, 'learning_rate': 3.208807785813777e-07, 'rewards/chosen': -0.9999314546585083, 'rewards/rejected': -2.8797202110290527, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.8797886371612549, 'logps/chosen': -159.43008422851562, 'logps/rejected': -452.0176696777344, 'logps/ref_chosen': -38.88483428955078, 'logps/ref_rejected': -104.38986206054688, 'logits/chosen': -2.008018970489502, 'logits/rejected': -1.9689502716064453, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.008289058692753315, 'epsilon_dpo/loss_margin_mean': 227.08258056640625, 'epsilon_dpo/beta_margin_mean': 1.8797887563705444, 'epsilon_dpo/beta_margin_std': 1.241491675376892, 'epsilon_dpo/beta_margin_grad_mean': -0.18520162999629974, 'epsilon_dpo/beta_margin_grad_std': 0.15284550189971924, 'kl/beta': 0.008361391723155975, 'kl/avg_steps': 0.875, 'epoch': 0.47}
|
|||
|
|
47%|█████████████████████████████████████████████████████▌ | 320/681 [19:55<14:56, 2.48s/it]
47%|█████████████████████████████████████████████████████▋ | 321/681 [19:57<15:05, 2.51s/it]
{'loss': 0.5089, 'grad_norm': 57.55246353149414, 'learning_rate': 3.1964918071004217e-07, 'rewards/chosen': -1.0699717998504639, 'rewards/rejected': -2.900449752807617, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.8304779529571533, 'logps/chosen': -163.596923828125, 'logps/rejected': -450.46417236328125, 'logps/ref_chosen': -33.60523986816406, 'logps/ref_rejected': -97.33091735839844, 'logits/chosen': -1.9321234226226807, 'logits/rejected': -1.7463536262512207, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.008217157796025276, 'epsilon_dpo/loss_margin_mean': 223.14154052734375, 'epsilon_dpo/beta_margin_mean': 1.8304779529571533, 'epsilon_dpo/beta_margin_std': 1.294824481010437, 'epsilon_dpo/beta_margin_grad_mean': -0.19574427604675293, 'epsilon_dpo/beta_margin_grad_std': 0.17788065969944, 'kl/beta': 0.00828886404633522, 'kl/avg_steps': 0.875, 'epoch': 0.47}
|
|||
|
|
47%|█████████████████████████████████████████████████████▋ | 321/681 [19:57<15:05, 2.51s/it]
47%|█████████████████████████████████████████████████████▉ | 322/681 [20:00<15:40, 2.62s/it]
{'loss': 0.4902, 'grad_norm': 51.22137451171875, 'learning_rate': 3.184157475180207e-07, 'rewards/chosen': -1.057375431060791, 'rewards/rejected': -2.8560502529144287, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.7986748218536377, 'logps/chosen': -169.1143798828125, 'logps/rejected': -452.2938537597656, 'logps/ref_chosen': -39.760765075683594, 'logps/ref_rejected': -101.8095703125, 'logits/chosen': -2.0422754287719727, 'logits/rejected': -1.863848090171814, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.008156154304742813, 'epsilon_dpo/loss_margin_mean': 221.13067626953125, 'epsilon_dpo/beta_margin_mean': 1.7986748218536377, 'epsilon_dpo/beta_margin_std': 1.202728509902954, 'epsilon_dpo/beta_margin_grad_mean': -0.1957569420337677, 'epsilon_dpo/beta_margin_grad_std': 0.16668935120105743, 'kl/beta': 0.008216965943574905, 'kl/avg_steps': 0.75, 'epoch': 0.47}
|
|||
|
|
47%|█████████████████████████████████████████████████████▉ | 322/681 [20:00<15:40, 2.62s/it]
47%|██████████████████████████████████████████████████████ | 323/681 [20:03<15:34, 2.61s/it]
{'loss': 0.5838, 'grad_norm': 59.20304489135742, 'learning_rate': 3.171805115074251e-07, 'rewards/chosen': -1.0898997783660889, 'rewards/rejected': -2.614929437637329, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.5250296592712402, 'logps/chosen': -173.9646759033203, 'logps/rejected': -402.1421203613281, 'logps/ref_chosen': -39.59267807006836, 'logps/ref_rejected': -78.83342742919922, 'logits/chosen': -1.9689267873764038, 'logits/rejected': -1.8758494853973389, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.008091602474451065, 'epsilon_dpo/loss_margin_mean': 188.9366912841797, 'epsilon_dpo/beta_margin_mean': 1.5250296592712402, 'epsilon_dpo/beta_margin_std': 1.1593869924545288, 'epsilon_dpo/beta_margin_grad_mean': -0.2292575240135193, 'epsilon_dpo/beta_margin_grad_std': 0.17154912650585175, 'kl/beta': 0.008155797608196735, 'kl/avg_steps': 0.796875, 'epoch': 0.47}
|
|||
|
|
47%|██████████████████████████████████████████████████████ | 323/681 [20:03<15:34, 2.61s/it]
48%|██████████████████████████████████████████████████████▏ | 324/681 [20:05<15:24, 2.59s/it]
{'loss': 0.4977, 'grad_norm': 56.6846923828125, 'learning_rate': 3.1594350522787295e-07, 'rewards/chosen': -0.9597259163856506, 'rewards/rejected': -2.9000043869018555, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.9402787685394287, 'logps/chosen': -158.4698486328125, 'logps/rejected': -455.7998962402344, 'logps/ref_chosen': -39.379974365234375, 'logps/ref_rejected': -94.40634155273438, 'logits/chosen': -2.086808443069458, 'logits/rejected': -1.915328025817871, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.008031437173485756, 'epsilon_dpo/loss_margin_mean': 242.30368041992188, 'epsilon_dpo/beta_margin_mean': 1.9402786493301392, 'epsilon_dpo/beta_margin_std': 1.3676698207855225, 'epsilon_dpo/beta_margin_grad_mean': -0.19054661691188812, 'epsilon_dpo/beta_margin_grad_std': 0.18834634125232697, 'kl/beta': 0.008091319352388382, 'kl/avg_steps': 0.75, 'epoch': 0.48}
|
|||
|
|
48%|██████████████████████████████████████████████████████▏ | 324/681 [20:05<15:24, 2.59s/it]
48%|██████████████████████████████████████████████████████▍ | 325/681 [20:08<15:18, 2.58s/it]
{'loss': 0.5678, 'grad_norm': 57.17675018310547, 'learning_rate': 3.147047612756302e-07, 'rewards/chosen': -1.032346248626709, 'rewards/rejected': -2.635953426361084, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.603606939315796, 'logps/chosen': -174.04409790039062, 'logps/rejected': -407.23272705078125, 'logps/ref_chosen': -44.78877258300781, 'logps/ref_rejected': -76.1741943359375, 'logits/chosen': -2.2039828300476074, 'logits/rejected': -1.9359973669052124, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.007969140075147152, 'epsilon_dpo/loss_margin_mean': 201.80323791503906, 'epsilon_dpo/beta_margin_mean': 1.603606939315796, 'epsilon_dpo/beta_margin_std': 1.2323416471481323, 'epsilon_dpo/beta_margin_grad_mean': -0.22008143365383148, 'epsilon_dpo/beta_margin_grad_std': 0.1740773767232895, 'kl/beta': 0.008031086064875126, 'kl/avg_steps': 0.78125, 'epoch': 0.48}
|
|||
|
|
48%|██████████████████████████████████████████████████████▍ | 325/681 [20:08<15:18, 2.58s/it]
48%|██████████████████████████████████████████████████████▌ | 326/681 [20:10<15:13, 2.57s/it]
{'loss': 0.4677, 'grad_norm': 43.5681037902832, 'learning_rate': 3.134643122927519e-07, 'rewards/chosen': -0.9895492792129517, 'rewards/rejected': -2.7935595512390137, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.8040101528167725, 'logps/chosen': -165.11163330078125, 'logps/rejected': -441.35858154296875, 'logps/ref_chosen': -39.91899871826172, 'logps/ref_rejected': -87.51998901367188, 'logits/chosen': -2.2271931171417236, 'logits/rejected': -1.8989125490188599, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.007899893447756767, 'epsilon_dpo/loss_margin_mean': 228.64596557617188, 'epsilon_dpo/beta_margin_mean': 1.8040101528167725, 'epsilon_dpo/beta_margin_std': 1.1729263067245483, 'epsilon_dpo/beta_margin_grad_mean': -0.19375857710838318, 'epsilon_dpo/beta_margin_grad_std': 0.14700661599636078, 'kl/beta': 0.007968829944729805, 'kl/avg_steps': 0.875, 'epoch': 0.48}
|
|||
|
|
48%|██████████████████████████████████████████████████████▌ | 326/681 [20:10<15:13, 2.57s/it]
48%|██████████████████████████████████████████████████████▋ | 327/681 [20:13<15:10, 2.57s/it]
{'loss': 0.3983, 'grad_norm': 45.13627624511719, 'learning_rate': 3.1222219096622264e-07, 'rewards/chosen': -0.8290703296661377, 'rewards/rejected': -2.890395164489746, 'rewards/accuracies': 0.9375, 'rewards/margins': 2.0613250732421875, 'logps/chosen': -143.63931274414062, 'logps/rejected': -489.22052001953125, 'logps/ref_chosen': -38.00127410888672, 'logps/ref_rejected': -119.96142578125, 'logits/chosen': -2.139697551727295, 'logits/rejected': -1.981135368347168, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.007831367664039135, 'epsilon_dpo/loss_margin_mean': 263.6210632324219, 'epsilon_dpo/beta_margin_mean': 2.0613248348236084, 'epsilon_dpo/beta_margin_std': 1.1618621349334717, 'epsilon_dpo/beta_margin_grad_mean': -0.16008856892585754, 'epsilon_dpo/beta_margin_grad_std': 0.15593233704566956, 'kl/beta': 0.007899707183241844, 'kl/avg_steps': 0.875, 'epoch': 0.48}
|
|||
|
|
48%|██████████████████████████████████████████████████████▋ | 327/681 [20:13<15:10, 2.57s/it]
48%|██████████████████████████████████████████████████████▉ | 328/681 [20:15<15:06, 2.57s/it]
{'loss': 0.4714, 'grad_norm': 43.36030197143555, 'learning_rate': 3.1097843002709427e-07, 'rewards/chosen': -0.9466592669487, 'rewards/rejected': -2.7843356132507324, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.8376761674880981, 'logps/chosen': -156.41744995117188, 'logps/rejected': -455.118408203125, 'logps/ref_chosen': -34.87682342529297, 'logps/ref_rejected': -96.5279769897461, 'logits/chosen': -2.1575894355773926, 'logits/rejected': -1.941756010055542, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0077707814052701, 'epsilon_dpo/loss_margin_mean': 237.0498046875, 'epsilon_dpo/beta_margin_mean': 1.8376761674880981, 'epsilon_dpo/beta_margin_std': 1.1939152479171753, 'epsilon_dpo/beta_margin_grad_mean': -0.18991978466510773, 'epsilon_dpo/beta_margin_grad_std': 0.1622304469347, 'kl/beta': 0.00783118512481451, 'kl/avg_steps': 0.78125, 'epoch': 0.48}
|
|||
|
|
48%|██████████████████████████████████████████████████████▉ | 328/681 [20:15<15:06, 2.57s/it]
48%|███████████████████████████████████████████████████████ | 329/681 [20:18<14:34, 2.48s/it]
{'loss': 0.4686, 'grad_norm': 50.19667053222656, 'learning_rate': 3.0973306224962437e-07, 'rewards/chosen': -0.8763324022293091, 'rewards/rejected': -2.7615880966186523, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.8852558135986328, 'logps/chosen': -148.80914306640625, 'logps/rejected': -463.7132568359375, 'logps/ref_chosen': -35.48017120361328, 'logps/ref_rejected': -105.25934600830078, 'logits/chosen': -2.229661464691162, 'logits/rejected': -2.011414051055908, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.007708113174885511, 'epsilon_dpo/loss_margin_mean': 245.12493896484375, 'epsilon_dpo/beta_margin_mean': 1.8852558135986328, 'epsilon_dpo/beta_margin_std': 1.2298924922943115, 'epsilon_dpo/beta_margin_grad_mean': -0.18650057911872864, 'epsilon_dpo/beta_margin_grad_std': 0.1677255630493164, 'kl/beta': 0.007770477794110775, 'kl/avg_steps': 0.8125, 'epoch': 0.48}
|
|||
|
|
48%|███████████████████████████████████████████████████████ | 329/681 [20:18<14:34, 2.48s/it]
48%|███████████████████████████████████████████████████████▏ | 330/681 [20:20<14:32, 2.49s/it]
{'loss': 0.4709, 'grad_norm': 45.98007583618164, 'learning_rate': 3.084861204504122e-07, 'rewards/chosen': -0.7483941912651062, 'rewards/rejected': -2.5697498321533203, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.8213555812835693, 'logps/chosen': -126.71240234375, 'logps/rejected': -429.37322998046875, 'logps/ref_chosen': -29.089645385742188, 'logps/ref_rejected': -93.00184631347656, 'logits/chosen': -2.0862879753112793, 'logits/rejected': -2.0670981407165527, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.007645989768207073, 'epsilon_dpo/loss_margin_mean': 238.7486572265625, 'epsilon_dpo/beta_margin_mean': 1.8213557004928589, 'epsilon_dpo/beta_margin_std': 1.1398029327392578, 'epsilon_dpo/beta_margin_grad_mean': -0.1875944435596466, 'epsilon_dpo/beta_margin_grad_std': 0.16437454521656036, 'kl/beta': 0.0077078514732420444, 'kl/avg_steps': 0.8125, 'epoch': 0.48}
|
|||
|
|
48%|███████████████████████████████████████████████████████▏ | 330/681 [20:20<14:32, 2.49s/it]
49%|███████████████████████████████████████████████████████▍ | 331/681 [20:23<14:37, 2.51s/it]
{'loss': 0.4442, 'grad_norm': 42.173667907714844, 'learning_rate': 3.072376374875335e-07, 'rewards/chosen': -0.8718706369400024, 'rewards/rejected': -2.720973014831543, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.84910249710083, 'logps/chosen': -145.30062866210938, 'logps/rejected': -454.640869140625, 'logps/ref_chosen': -30.57064437866211, 'logps/ref_rejected': -95.55513000488281, 'logits/chosen': -2.128115177154541, 'logits/rejected': -1.9702262878417969, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0075819771736860275, 'epsilon_dpo/loss_margin_mean': 244.35574340820312, 'epsilon_dpo/beta_margin_mean': 1.8491023778915405, 'epsilon_dpo/beta_margin_std': 1.0536081790924072, 'epsilon_dpo/beta_margin_grad_mean': -0.1762796938419342, 'epsilon_dpo/beta_margin_grad_std': 0.15704338252544403, 'kl/beta': 0.0076457299292087555, 'kl/avg_steps': 0.84375, 'epoch': 0.49}
|
|||
|
|
49%|███████████████████████████████████████████████████████▍ | 331/681 [20:23<14:37, 2.51s/it]
49%|███████████████████████████████████████████████████████▌ | 332/681 [20:25<14:27, 2.49s/it]
{'loss': 0.5063, 'grad_norm': 46.4879035949707, 'learning_rate': 3.059876462596758e-07, 'rewards/chosen': -0.8930763006210327, 'rewards/rejected': -2.554720163345337, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.6616438627243042, 'logps/chosen': -150.6046142578125, 'logps/rejected': -421.76385498046875, 'logps/ref_chosen': -31.9512882232666, 'logps/ref_rejected': -81.75474548339844, 'logits/chosen': -2.075554370880127, 'logits/rejected': -1.9067268371582031, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.007520908955484629, 'epsilon_dpo/loss_margin_mean': 221.3557891845703, 'epsilon_dpo/beta_margin_mean': 1.6616438627243042, 'epsilon_dpo/beta_margin_std': 1.0939134359359741, 'epsilon_dpo/beta_margin_grad_mean': -0.2063533514738083, 'epsilon_dpo/beta_margin_grad_std': 0.1539819985628128, 'kl/beta': 0.00758175877854228, 'kl/avg_steps': 0.8125, 'epoch': 0.49}
|
|||
|
|
49%|███████████████████████████████████████████████████████▌ | 332/681 [20:25<14:27, 2.49s/it]
49%|███████████████████████████████████████████████████████▋ | 333/681 [20:28<14:27, 2.49s/it]
{'loss': 0.4612, 'grad_norm': 41.35095977783203, 'learning_rate': 3.0473617970527015e-07, 'rewards/chosen': -0.8483164310455322, 'rewards/rejected': -2.608299732208252, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.7599835395812988, 'logps/chosen': -148.9061737060547, 'logps/rejected': -452.73358154296875, 'logps/ref_chosen': -35.37367630004883, 'logps/ref_rejected': -102.79765319824219, 'logits/chosen': -2.1496317386627197, 'logits/rejected': -2.04693603515625, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.007457944098860025, 'epsilon_dpo/loss_margin_mean': 236.4034423828125, 'epsilon_dpo/beta_margin_mean': 1.7599835395812988, 'epsilon_dpo/beta_margin_std': 1.0482655763626099, 'epsilon_dpo/beta_margin_grad_mean': -0.1893877238035202, 'epsilon_dpo/beta_margin_grad_std': 0.14721722900867462, 'kl/beta': 0.007520653773099184, 'kl/avg_steps': 0.84375, 'epoch': 0.49}
|
|||
|
|
49%|███████████████████████████████████████████████████████▋ | 333/681 [20:28<14:27, 2.49s/it]
49%|███████████████████████████████████████████████████████▉ | 334/681 [20:30<14:36, 2.53s/it]
{'loss': 0.5177, 'grad_norm': 43.43370819091797, 'learning_rate': 3.034832708016243e-07, 'rewards/chosen': -0.8658192157745361, 'rewards/rejected': -2.503511428833008, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.6376922130584717, 'logps/chosen': -160.87864685058594, 'logps/rejected': -440.5398864746094, 'logps/ref_chosen': -44.046653747558594, 'logps/ref_rejected': -101.81437683105469, 'logits/chosen': -2.1904244422912598, 'logits/rejected': -2.034341812133789, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.007395543623715639, 'epsilon_dpo/loss_margin_mean': 221.89352416992188, 'epsilon_dpo/beta_margin_mean': 1.6376922130584717, 'epsilon_dpo/beta_margin_std': 1.04725182056427, 'epsilon_dpo/beta_margin_grad_mean': -0.20312325656414032, 'epsilon_dpo/beta_margin_grad_std': 0.1642070859670639, 'kl/beta': 0.007457728963345289, 'kl/avg_steps': 0.84375, 'epoch': 0.49}
|
|||
|
|
49%|███████████████████████████████████████████████████████▉ | 334/681 [20:30<14:36, 2.53s/it]
49%|████████████████████████████████████████████████████████ | 335/681 [20:33<14:45, 2.56s/it]
{'loss': 0.6057, 'grad_norm': 51.230499267578125, 'learning_rate': 3.022289525640531e-07, 'rewards/chosen': -1.0017235279083252, 'rewards/rejected': -2.3570237159729004, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.3553001880645752, 'logps/chosen': -182.1242218017578, 'logps/rejected': -414.8521728515625, 'logps/ref_chosen': -45.84385681152344, 'logps/ref_rejected': -93.38748931884766, 'logits/chosen': -2.224094867706299, 'logits/rejected': -2.0437393188476562, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.007335977163165808, 'epsilon_dpo/loss_margin_mean': 185.18431091308594, 'epsilon_dpo/beta_margin_mean': 1.3553001880645752, 'epsilon_dpo/beta_margin_std': 0.970168948173523, 'epsilon_dpo/beta_margin_grad_mean': -0.24230733513832092, 'epsilon_dpo/beta_margin_grad_std': 0.15432856976985931, 'kl/beta': 0.0073953308165073395, 'kl/avg_steps': 0.8125, 'epoch': 0.49}
|
|||
|
|
49%|████████████████████████████████████████████████████████ | 335/681 [20:33<14:45, 2.56s/it]
49%|████████████████████████████████████████████████████████▏ | 336/681 [20:35<14:26, 2.51s/it]
{'loss': 0.4338, 'grad_norm': 50.833736419677734, 'learning_rate': 3.009732580450086e-07, 'rewards/chosen': -0.8097207546234131, 'rewards/rejected': -2.6393799781799316, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.829659104347229, 'logps/chosen': -138.39865112304688, 'logps/rejected': -476.6292419433594, 'logps/ref_chosen': -27.110448837280273, 'logps/ref_rejected': -113.38800048828125, 'logits/chosen': -2.130596876144409, 'logits/rejected': -2.1031112670898438, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.00726768234744668, 'epsilon_dpo/loss_margin_mean': 251.95303344726562, 'epsilon_dpo/beta_margin_mean': 1.829659104347229, 'epsilon_dpo/beta_margin_std': 1.08203125, 'epsilon_dpo/beta_margin_grad_mean': -0.18038199841976166, 'epsilon_dpo/beta_margin_grad_std': 0.13710618019104004, 'kl/beta': 0.007335728034377098, 'kl/avg_steps': 0.9375, 'epoch': 0.49}
|
|||
|
|
49%|████████████████████████████████████████████████████████▏ | 336/681 [20:35<14:26, 2.51s/it]
49%|████████████████████████████████████████████████████████▍ | 337/681 [20:38<14:00, 2.44s/it]
{'loss': 0.5162, 'grad_norm': 54.00065612792969, 'learning_rate': 2.9971622033320914e-07, 'rewards/chosen': -0.9518444538116455, 'rewards/rejected': -2.663625717163086, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.7117812633514404, 'logps/chosen': -177.97787475585938, 'logps/rejected': -479.04248046875, 'logps/ref_chosen': -46.36784362792969, 'logps/ref_rejected': -109.42700958251953, 'logits/chosen': -2.1570067405700684, 'logits/rejected': -2.0568230152130127, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.007211537566035986, 'epsilon_dpo/loss_margin_mean': 238.00546264648438, 'epsilon_dpo/beta_margin_mean': 1.7117812633514404, 'epsilon_dpo/beta_margin_std': 1.1512197256088257, 'epsilon_dpo/beta_margin_grad_mean': -0.2015434354543686, 'epsilon_dpo/beta_margin_grad_std': 0.17217382788658142, 'kl/beta': 0.007267594337463379, 'kl/avg_steps': 0.78125, 'epoch': 0.49}
|
|||
|
|
49%|████████████████████████████████████████████████████████▍ | 337/681 [20:38<14:00, 2.44s/it]
50%|████████████████████████████████████████████████████████▌ | 338/681 [20:40<13:58, 2.44s/it]
{'loss': 0.5573, 'grad_norm': 54.01927185058594, 'learning_rate': 2.984578725527675e-07, 'rewards/chosen': -0.9232571125030518, 'rewards/rejected': -2.454974412918091, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.531717300415039, 'logps/chosen': -169.5230712890625, 'logps/rejected': -437.54058837890625, 'logps/ref_chosen': -40.62709045410156, 'logps/ref_rejected': -94.02056121826172, 'logits/chosen': -2.093170166015625, 'logits/rejected': -2.0884156227111816, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.007151126395910978, 'epsilon_dpo/loss_margin_mean': 214.6240692138672, 'epsilon_dpo/beta_margin_mean': 1.531717300415039, 'epsilon_dpo/beta_margin_std': 1.0485196113586426, 'epsilon_dpo/beta_margin_grad_mean': -0.21953918039798737, 'epsilon_dpo/beta_margin_grad_std': 0.16535867750644684, 'kl/beta': 0.007211256306618452, 'kl/avg_steps': 0.84375, 'epoch': 0.5}
|
|||
|
|
50%|████████████████████████████████████████████████████████▌ | 338/681 [20:40<13:58, 2.44s/it]
50%|████████████████████████████████████████████████████████▋ | 339/681 [20:42<13:46, 2.42s/it]
{'loss': 0.4775, 'grad_norm': 48.617767333984375, 'learning_rate': 2.9719824786231796e-07, 'rewards/chosen': -0.9160549640655518, 'rewards/rejected': -2.7034502029418945, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.7873952388763428, 'logps/chosen': -171.6763458251953, 'logps/rejected': -495.22967529296875, 'logps/ref_chosen': -42.66804885864258, 'logps/ref_rejected': -113.68071746826172, 'logits/chosen': -2.1362557411193848, 'logits/rejected': -2.0298609733581543, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.007089058868587017, 'epsilon_dpo/loss_margin_mean': 252.5406494140625, 'epsilon_dpo/beta_margin_mean': 1.7873952388763428, 'epsilon_dpo/beta_margin_std': 1.163671851158142, 'epsilon_dpo/beta_margin_grad_mean': -0.19239673018455505, 'epsilon_dpo/beta_margin_grad_std': 0.1555846780538559, 'kl/beta': 0.007150920573621988, 'kl/avg_steps': 0.875, 'epoch': 0.5}
|
|||
|
|
50%|████████████████████████████████████████████████████████▋ | 339/681 [20:42<13:46, 2.42s/it]
50%|████████████████████████████████████████████████████████▉ | 340/681 [20:45<13:30, 2.38s/it]
{'loss': 0.6076, 'grad_norm': 53.56328582763672, 'learning_rate': 2.959373794541426e-07, 'rewards/chosen': -0.8387173414230347, 'rewards/rejected': -2.3515944480895996, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.5128772258758545, 'logps/chosen': -152.33245849609375, 'logps/rejected': -413.41455078125, 'logps/ref_chosen': -33.344947814941406, 'logps/ref_rejected': -78.67686462402344, 'logits/chosen': -2.1749508380889893, 'logits/rejected': -1.981499195098877, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.007029782980680466, 'epsilon_dpo/loss_margin_mean': 215.7501983642578, 'epsilon_dpo/beta_margin_mean': 1.512877106666565, 'epsilon_dpo/beta_margin_std': 1.1476131677627563, 'epsilon_dpo/beta_margin_grad_mean': -0.22466416656970978, 'epsilon_dpo/beta_margin_grad_std': 0.18301187455654144, 'kl/beta': 0.007088892627507448, 'kl/avg_steps': 0.84375, 'epoch': 0.5}
|
|||
|
|
50%|████████████████████████████████████████████████████████▉ | 340/681 [20:45<13:30, 2.38s/it]
50%|█████████████████████████████████████████████████████████ | 341/681 [20:47<13:37, 2.40s/it]
{'loss': 0.5095, 'grad_norm': 51.781864166259766, 'learning_rate': 2.946753005532965e-07, 'rewards/chosen': -0.7515710592269897, 'rewards/rejected': -2.5373077392578125, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.7857365608215332, 'logps/chosen': -133.99024963378906, 'logps/rejected': -472.0899353027344, 'logps/ref_chosen': -26.7175235748291, 'logps/ref_rejected': -108.1113510131836, 'logits/chosen': -2.1569323539733887, 'logits/rejected': -1.9963412284851074, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.006977556273341179, 'epsilon_dpo/loss_margin_mean': 256.70587158203125, 'epsilon_dpo/beta_margin_mean': 1.7857365608215332, 'epsilon_dpo/beta_margin_std': 1.1933374404907227, 'epsilon_dpo/beta_margin_grad_mean': -0.1975635439157486, 'epsilon_dpo/beta_margin_grad_std': 0.181951105594635, 'kl/beta': 0.007029580418020487, 'kl/avg_steps': 0.75, 'epoch': 0.5}
|
|||
|
|
50%|█████████████████████████████████████████████████████████ | 341/681 [20:47<13:37, 2.40s/it]
50%|█████████████████████████████████████████████████████████▎ | 342/681 [20:50<13:54, 2.46s/it]
{'loss': 0.4569, 'grad_norm': 45.30876159667969, 'learning_rate': 2.934120444167326e-07, 'rewards/chosen': -0.8333628177642822, 'rewards/rejected': -2.640061378479004, 'rewards/accuracies': 0.984375, 'rewards/margins': 1.8066987991333008, 'logps/chosen': -175.69277954101562, 'logps/rejected': -482.6606140136719, 'logps/ref_chosen': -55.37556838989258, 'logps/ref_rejected': -100.79505920410156, 'logits/chosen': -2.289546489715576, 'logits/rejected': -1.9754924774169922, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.006912530865520239, 'epsilon_dpo/loss_margin_mean': 261.54833984375, 'epsilon_dpo/beta_margin_mean': 1.8066987991333008, 'epsilon_dpo/beta_margin_std': 1.138454794883728, 'epsilon_dpo/beta_margin_grad_mean': -0.18946228921413422, 'epsilon_dpo/beta_margin_grad_std': 0.1443195641040802, 'kl/beta': 0.006977251265197992, 'kl/avg_steps': 0.9375, 'epoch': 0.5}
|
|||
|
|
50%|█████████████████████████████████████████████████████████▎ | 342/681 [20:50<13:54, 2.46s/it]
50%|█████████████████████████████████████████████████████████▍ | 343/681 [20:52<13:43, 2.44s/it]
{'loss': 0.514, 'grad_norm': 49.670379638671875, 'learning_rate': 2.9214764433242476e-07, 'rewards/chosen': -0.9355216026306152, 'rewards/rejected': -2.3873019218444824, 'rewards/accuracies': 0.984375, 'rewards/margins': 1.451780080795288, 'logps/chosen': -178.66543579101562, 'logps/rejected': -466.6171569824219, 'logps/ref_chosen': -42.08940124511719, 'logps/ref_rejected': -117.90960693359375, 'logits/chosen': -2.164459228515625, 'logits/rejected': -2.1661453247070312, 'kl/p_epsilon_steps': 0.984375, 'kl/n_epsilon_steps': 0.015625, 'epsilon_dpo/beta': 0.006846167147159576, 'epsilon_dpo/loss_margin_mean': 212.13150024414062, 'epsilon_dpo/beta_margin_mean': 1.4517802000045776, 'epsilon_dpo/beta_margin_std': 0.8032383322715759, 'epsilon_dpo/beta_margin_grad_mean': -0.21721310913562775, 'epsilon_dpo/beta_margin_grad_std': 0.11748269945383072, 'kl/beta': 0.0069124470464885235, 'kl/avg_steps': 0.96875, 'epoch': 0.5}
|
|||
|
|
50%|█████████████████████████████████████████████████████████▍ | 343/681 [20:52<13:43, 2.44s/it]
51%|█████████████████████████████████████████████████████████▌ | 344/681 [20:55<13:46, 2.45s/it]
{'loss': 0.6315, 'grad_norm': 57.320960998535156, 'learning_rate': 2.9088213361849126e-07, 'rewards/chosen': -0.8735264539718628, 'rewards/rejected': -2.325773239135742, 'rewards/accuracies': 0.859375, 'rewards/margins': 1.4522466659545898, 'logps/chosen': -162.51026916503906, 'logps/rejected': -439.02960205078125, 'logps/ref_chosen': -34.435752868652344, 'logps/ref_rejected': -96.5870361328125, 'logits/chosen': -2.179155111312866, 'logits/rejected': -2.0898489952087402, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.006797598209232092, 'epsilon_dpo/loss_margin_mean': 214.36805725097656, 'epsilon_dpo/beta_margin_mean': 1.4522466659545898, 'epsilon_dpo/beta_margin_std': 1.2337263822555542, 'epsilon_dpo/beta_margin_grad_mean': -0.24506476521492004, 'epsilon_dpo/beta_margin_grad_std': 0.1786363273859024, 'kl/beta': 0.006846125237643719, 'kl/avg_steps': 0.71875, 'epoch': 0.51}
|
|||
|
|
51%|█████████████████████████████████████████████████████████▌ | 344/681 [20:55<13:46, 2.45s/it]
51%|█████████████████████████████████████████████████████████▊ | 345/681 [20:57<14:07, 2.52s/it]
{'loss': 0.4414, 'grad_norm': 50.00996398925781, 'learning_rate': 2.896155456223163e-07, 'rewards/chosen': -0.8974334001541138, 'rewards/rejected': -2.6806066036224365, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.7831730842590332, 'logps/chosen': -173.568603515625, 'logps/rejected': -505.9549560546875, 'logps/ref_chosen': -40.498329162597656, 'logps/ref_rejected': -107.90937805175781, 'logits/chosen': -2.24761962890625, 'logits/rejected': -2.086604595184326, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0067384676076471806, 'epsilon_dpo/loss_margin_mean': 264.975341796875, 'epsilon_dpo/beta_margin_mean': 1.7831730842590332, 'epsilon_dpo/beta_margin_std': 1.0325002670288086, 'epsilon_dpo/beta_margin_grad_mean': -0.18444088101387024, 'epsilon_dpo/beta_margin_grad_std': 0.13682545721530914, 'kl/beta': 0.00679726991802454, 'kl/avg_steps': 0.875, 'epoch': 0.51}
|
|||
|
|
51%|█████████████████████████████████████████████████████████▊ | 345/681 [20:57<14:07, 2.52s/it]
51%|█████████████████████████████████████████████████████████▉ | 346/681 [21:00<14:04, 2.52s/it]
{'loss': 0.4511, 'grad_norm': 42.087677001953125, 'learning_rate': 2.883479137196714e-07, 'rewards/chosen': -0.8508467078208923, 'rewards/rejected': -2.5596466064453125, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.7088000774383545, 'logps/chosen': -160.73776245117188, 'logps/rejected': -467.2408447265625, 'logps/ref_chosen': -33.41410827636719, 'logps/ref_rejected': -83.742919921875, 'logits/chosen': -2.0703492164611816, 'logits/rejected': -1.9229077100753784, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.00667791161686182, 'epsilon_dpo/loss_margin_mean': 256.1742858886719, 'epsilon_dpo/beta_margin_mean': 1.7088000774383545, 'epsilon_dpo/beta_margin_std': 0.9547905325889587, 'epsilon_dpo/beta_margin_grad_mean': -0.18874618411064148, 'epsilon_dpo/beta_margin_grad_std': 0.13201214373111725, 'kl/beta': 0.006738309748470783, 'kl/avg_steps': 0.90625, 'epoch': 0.51}
|
|||
|
|
51%|█████████████████████████████████████████████████████████▉ | 346/681 [21:00<14:04, 2.52s/it]
51%|██████████████████████████████████████████████████████████ | 347/681 [21:02<13:45, 2.47s/it]
{'loss': 0.7379, 'grad_norm': 72.8255615234375, 'learning_rate': 2.8707927131383614e-07, 'rewards/chosen': -1.1159558296203613, 'rewards/rejected': -2.416703701019287, 'rewards/accuracies': 0.875, 'rewards/margins': 1.3007481098175049, 'logps/chosen': -213.68258666992188, 'logps/rejected': -461.857421875, 'logps/ref_chosen': -46.14110565185547, 'logps/ref_rejected': -97.28417205810547, 'logits/chosen': -2.1761817932128906, 'logits/rejected': -2.0537772178649902, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.006634632125496864, 'epsilon_dpo/loss_margin_mean': 197.03175354003906, 'epsilon_dpo/beta_margin_mean': 1.3007481098175049, 'epsilon_dpo/beta_margin_std': 1.2315279245376587, 'epsilon_dpo/beta_margin_grad_mean': -0.26513612270355225, 'epsilon_dpo/beta_margin_grad_std': 0.2060505747795105, 'kl/beta': 0.006677791941910982, 'kl/avg_steps': 0.65625, 'epoch': 0.51}
|
|||
|
|
51%|██████████████████████████████████████████████████████████ | 347/681 [21:02<13:45, 2.47s/it]
51%|██████████████████████████████████████████████████████████▎ | 348/681 [21:05<13:57, 2.51s/it]
{'loss': 0.5108, 'grad_norm': 40.67171859741211, 'learning_rate': 2.858096518347179e-07, 'rewards/chosen': -0.8284679651260376, 'rewards/rejected': -2.446683645248413, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.618215560913086, 'logps/chosen': -162.21255493164062, 'logps/rejected': -466.0066833496094, 'logps/ref_chosen': -36.474021911621094, 'logps/ref_rejected': -93.86297607421875, 'logits/chosen': -2.202709674835205, 'logits/rejected': -2.092670440673828, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0065768626518547535, 'epsilon_dpo/loss_margin_mean': 246.40516662597656, 'epsilon_dpo/beta_margin_mean': 1.618215560913086, 'epsilon_dpo/beta_margin_std': 1.0140705108642578, 'epsilon_dpo/beta_margin_grad_mean': -0.20659483969211578, 'epsilon_dpo/beta_margin_grad_std': 0.15565478801727295, 'kl/beta': 0.006634254939854145, 'kl/avg_steps': 0.875, 'epoch': 0.51}
|
|||
|
|
51%|██████████████████████████████████████████████████████████▎ | 348/681 [21:05<13:57, 2.51s/it]
51%|██████████████████████████████████████████████████████████▍ | 349/681 [21:07<13:54, 2.51s/it]
{'loss': 0.5577, 'grad_norm': 54.361026763916016, 'learning_rate': 2.845390887379706e-07, 'rewards/chosen': -0.8179513216018677, 'rewards/rejected': -2.4374313354492188, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.6194798946380615, 'logps/chosen': -157.87994384765625, 'logps/rejected': -478.4164733886719, 'logps/ref_chosen': -32.72327423095703, 'logps/ref_rejected': -104.45381927490234, 'logits/chosen': -2.1096177101135254, 'logits/rejected': -2.068809986114502, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.006519814487546682, 'epsilon_dpo/loss_margin_mean': 248.80596923828125, 'epsilon_dpo/beta_margin_mean': 1.6194798946380615, 'epsilon_dpo/beta_margin_std': 1.1991218328475952, 'epsilon_dpo/beta_margin_grad_mean': -0.2202288955450058, 'epsilon_dpo/beta_margin_grad_std': 0.171220600605011, 'kl/beta': 0.006576708517968655, 'kl/avg_steps': 0.875, 'epoch': 0.51}
|
|||
|
|
51%|██████████████████████████████████████████████████████████▍ | 349/681 [21:07<13:54, 2.51s/it]
51%|██████████████████████████████████████████████████████████▌ | 350/681 [21:10<13:51, 2.51s/it]
{'loss': 0.6014, 'grad_norm': 66.0088119506836, 'learning_rate': 2.8326761550411346e-07, 'rewards/chosen': -0.9508112668991089, 'rewards/rejected': -2.4584474563598633, 'rewards/accuracies': 0.875, 'rewards/margins': 1.507636308670044, 'logps/chosen': -188.4402618408203, 'logps/rejected': -477.0697326660156, 'logps/ref_chosen': -41.924171447753906, 'logps/ref_rejected': -96.81851196289062, 'logits/chosen': -2.133009910583496, 'logits/rejected': -2.081470489501953, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.006471411325037479, 'epsilon_dpo/loss_margin_mean': 233.73513793945312, 'epsilon_dpo/beta_margin_mean': 1.507636308670044, 'epsilon_dpo/beta_margin_std': 1.1591410636901855, 'epsilon_dpo/beta_margin_grad_mean': -0.2324196696281433, 'epsilon_dpo/beta_margin_grad_std': 0.18096159398555756, 'kl/beta': 0.006519661750644445, 'kl/avg_steps': 0.75, 'epoch': 0.51}
|
|||
|
|
51%|██████████████████████████████████████████████████████████▌ | 350/681 [21:10<13:51, 2.51s/it]
52%|██████████████████████████████████████████████████████████▊ | 351/681 [21:12<13:54, 2.53s/it]
{'loss': 0.5544, 'grad_norm': 63.894622802734375, 'learning_rate': 2.819952656376487e-07, 'rewards/chosen': -0.9005885124206543, 'rewards/rejected': -2.4676437377929688, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.5670552253723145, 'logps/chosen': -178.15701293945312, 'logps/rejected': -494.2790222167969, 'logps/ref_chosen': -38.231632232666016, 'logps/ref_rejected': -109.67044830322266, 'logits/chosen': -2.141003131866455, 'logits/rejected': -2.1702256202697754, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.006419192533940077, 'epsilon_dpo/loss_margin_mean': 244.6831817626953, 'epsilon_dpo/beta_margin_mean': 1.5670552253723145, 'epsilon_dpo/beta_margin_std': 1.1509732007980347, 'epsilon_dpo/beta_margin_grad_mean': -0.22091074287891388, 'epsilon_dpo/beta_margin_grad_std': 0.16051997244358063, 'kl/beta': 0.006471128202974796, 'kl/avg_steps': 0.8125, 'epoch': 0.52}
|
|||
|
|
52%|██████████████████████████████████████████████████████████▊ | 351/681 [21:12<13:54, 2.53s/it]
52%|██████████████████████████████████████████████████████████▉ | 352/681 [21:15<13:53, 2.53s/it]
{'loss': 0.5649, 'grad_norm': 50.314727783203125, 'learning_rate': 2.8072207266617854e-07, 'rewards/chosen': -0.8179040551185608, 'rewards/rejected': -2.3090052604675293, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.4911013841629028, 'logps/chosen': -167.52468872070312, 'logps/rejected': -445.0927734375, 'logps/ref_chosen': -39.300880432128906, 'logps/ref_rejected': -82.15660095214844, 'logits/chosen': -2.2193918228149414, 'logits/rejected': -2.0443153381347656, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.006367456633597612, 'epsilon_dpo/loss_margin_mean': 234.71234130859375, 'epsilon_dpo/beta_margin_mean': 1.4911013841629028, 'epsilon_dpo/beta_margin_std': 1.0576071739196777, 'epsilon_dpo/beta_margin_grad_mean': -0.22667014598846436, 'epsilon_dpo/beta_margin_grad_std': 0.1572793573141098, 'kl/beta': 0.0064189741387963295, 'kl/avg_steps': 0.8125, 'epoch': 0.52}
|
|||
|
|
52%|██████████████████████████████████████████████████████████▉ | 352/681 [21:15<13:53, 2.53s/it]
52%|███████████████████████████████████████████████████████████ | 353/681 [21:17<13:42, 2.51s/it]
{'loss': 0.6444, 'grad_norm': 57.24125289916992, 'learning_rate': 2.794480701395219e-07, 'rewards/chosen': -0.9420816898345947, 'rewards/rejected': -2.368312120437622, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.4262304306030273, 'logps/chosen': -186.15036010742188, 'logps/rejected': -461.47979736328125, 'logps/ref_chosen': -37.74017333984375, 'logps/ref_rejected': -86.70198059082031, 'logits/chosen': -2.104984760284424, 'logits/rejected': -2.00747013092041, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.006326087284833193, 'epsilon_dpo/loss_margin_mean': 226.36761474609375, 'epsilon_dpo/beta_margin_mean': 1.4262304306030273, 'epsilon_dpo/beta_margin_std': 1.1893874406814575, 'epsilon_dpo/beta_margin_grad_mean': -0.24461467564105988, 'epsilon_dpo/beta_margin_grad_std': 0.182557612657547, 'kl/beta': 0.006367240101099014, 'kl/avg_steps': 0.65625, 'epoch': 0.52}
|
|||
|
|
52%|███████████████████████████████████████████████████████████ | 353/681 [21:17<13:42, 2.51s/it]
52%|███████████████████████████████████████████████████████████▎ | 354/681 [21:20<13:20, 2.45s/it]
{'loss': 0.5369, 'grad_norm': 44.26091384887695, 'learning_rate': 2.781732916288303e-07, 'rewards/chosen': -0.8998274803161621, 'rewards/rejected': -2.382763147354126, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.4829356670379639, 'logps/chosen': -189.85113525390625, 'logps/rejected': -474.8692626953125, 'logps/ref_chosen': -46.64361572265625, 'logps/ref_rejected': -94.82032775878906, 'logits/chosen': -2.21701717376709, 'logits/rejected': -2.05487322807312, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.006276935804635286, 'epsilon_dpo/loss_margin_mean': 236.8414306640625, 'epsilon_dpo/beta_margin_mean': 1.4829356670379639, 'epsilon_dpo/beta_margin_std': 0.9004656672477722, 'epsilon_dpo/beta_margin_grad_mean': -0.21808402240276337, 'epsilon_dpo/beta_margin_grad_std': 0.14849531650543213, 'kl/beta': 0.006325727794319391, 'kl/avg_steps': 0.78125, 'epoch': 0.52}
|
|||
|
|
52%|███████████████████████████████████████████████████████████▎ | 354/681 [21:20<13:20, 2.45s/it]
52%|███████████████████████████████████████████████████████████▍ | 355/681 [21:22<13:22, 2.46s/it]
{'loss': 0.473, 'grad_norm': 43.440181732177734, 'learning_rate': 2.7689777072570284e-07, 'rewards/chosen': -0.7434677481651306, 'rewards/rejected': -2.3702354431152344, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.626767873764038, 'logps/chosen': -158.43057250976562, 'logps/rejected': -469.4125061035156, 'logps/ref_chosen': -38.99018859863281, 'logps/ref_rejected': -88.16902160644531, 'logits/chosen': -2.233151435852051, 'logits/rejected': -2.0062508583068848, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.006220430601388216, 'epsilon_dpo/loss_margin_mean': 261.8030700683594, 'epsilon_dpo/beta_margin_mean': 1.626767873764038, 'epsilon_dpo/beta_margin_std': 0.9073279500007629, 'epsilon_dpo/beta_margin_grad_mean': -0.19742940366268158, 'epsilon_dpo/beta_margin_grad_std': 0.1338631510734558, 'kl/beta': 0.0062766908667981625, 'kl/avg_steps': 0.90625, 'epoch': 0.52}
|
|||
|
|
52%|███████████████████████████████████████████████████████████▍ | 355/681 [21:22<13:22, 2.46s/it]
52%|███████████████████████████████████████████████████████████▌ | 356/681 [21:25<13:50, 2.56s/it]
{'loss': 0.6355, 'grad_norm': 52.8966178894043, 'learning_rate': 2.7562154104130176e-07, 'rewards/chosen': -0.8773345351219177, 'rewards/rejected': -2.2625231742858887, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.3851888179779053, 'logps/chosen': -177.79510498046875, 'logps/rejected': -447.9184265136719, 'logps/ref_chosen': -36.000328063964844, 'logps/ref_rejected': -81.04385375976562, 'logits/chosen': -2.149020195007324, 'logits/rejected': -1.9297568798065186, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.006174284033477306, 'epsilon_dpo/loss_margin_mean': 225.0797882080078, 'epsilon_dpo/beta_margin_mean': 1.3851886987686157, 'epsilon_dpo/beta_margin_std': 1.0974996089935303, 'epsilon_dpo/beta_margin_grad_mean': -0.2459879219532013, 'epsilon_dpo/beta_margin_grad_std': 0.1747446060180664, 'kl/beta': 0.0062203193083405495, 'kl/avg_steps': 0.75, 'epoch': 0.52}
|
|||
|
|
52%|███████████████████████████████████████████████████████████▌ | 356/681 [21:25<13:50, 2.56s/it]
52%|███████████████████████████████████████████████████████████▊ | 357/681 [21:27<13:53, 2.57s/it]
{'loss': 0.5852, 'grad_norm': 53.406307220458984, 'learning_rate': 2.7434463620546594e-07, 'rewards/chosen': -0.8109502792358398, 'rewards/rejected': -2.3253703117370605, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.5144197940826416, 'logps/chosen': -169.40367126464844, 'logps/rejected': -470.9423522949219, 'logps/ref_chosen': -37.324180603027344, 'logps/ref_rejected': -90.94546508789062, 'logits/chosen': -2.1703100204467773, 'logits/rejected': -2.005626916885376, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.006120603997260332, 'epsilon_dpo/loss_margin_mean': 247.9174041748047, 'epsilon_dpo/beta_margin_mean': 1.5144197940826416, 'epsilon_dpo/beta_margin_std': 1.1098681688308716, 'epsilon_dpo/beta_margin_grad_mean': -0.22637690603733063, 'epsilon_dpo/beta_margin_grad_std': 0.17489291727542877, 'kl/beta': 0.006174014415591955, 'kl/avg_steps': 0.875, 'epoch': 0.52}
|
|||
|
|
52%|███████████████████████████████████████████████████████████▊ | 357/681 [21:28<13:53, 2.57s/it]
53%|███████████████████████████████████████████████████████████▉ | 358/681 [21:30<13:55, 2.59s/it]
{'loss': 0.6267, 'grad_norm': 54.73785400390625, 'learning_rate': 2.730670898658255e-07, 'rewards/chosen': -0.7564330101013184, 'rewards/rejected': -2.1590704917907715, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.4026376008987427, 'logps/chosen': -169.3449249267578, 'logps/rejected': -450.82177734375, 'logps/ref_chosen': -45.07238006591797, 'logps/ref_rejected': -94.94227600097656, 'logits/chosen': -2.2487030029296875, 'logits/rejected': -2.109694480895996, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0060713388957083225, 'epsilon_dpo/loss_margin_mean': 231.60694885253906, 'epsilon_dpo/beta_margin_mean': 1.4026376008987427, 'epsilon_dpo/beta_margin_std': 1.0886473655700684, 'epsilon_dpo/beta_margin_grad_mean': -0.2422979176044464, 'epsilon_dpo/beta_margin_grad_std': 0.17289938032627106, 'kl/beta': 0.006120460107922554, 'kl/avg_steps': 0.8125, 'epoch': 0.53}
|
|||
|
|
53%|███████████████████████████████████████████████████████████▉ | 358/681 [21:30<13:55, 2.59s/it]
53%|████████████████████████████████████████████████████████████ | 359/681 [21:33<13:36, 2.54s/it]
{'loss': 0.6242, 'grad_norm': 62.71131134033203, 'learning_rate': 2.717889356869146e-07, 'rewards/chosen': -0.7723500728607178, 'rewards/rejected': -2.258676052093506, 'rewards/accuracies': 0.875, 'rewards/margins': 1.486325979232788, 'logps/chosen': -162.13319396972656, 'logps/rejected': -462.682373046875, 'logps/ref_chosen': -34.454952239990234, 'logps/ref_rejected': -87.54586791992188, 'logits/chosen': -2.1315059661865234, 'logits/rejected': -1.9415364265441895, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0060280985198915005, 'epsilon_dpo/loss_margin_mean': 247.45826721191406, 'epsilon_dpo/beta_margin_mean': 1.4863260984420776, 'epsilon_dpo/beta_margin_std': 1.1895928382873535, 'epsilon_dpo/beta_margin_grad_mean': -0.23849013447761536, 'epsilon_dpo/beta_margin_grad_std': 0.1872168630361557, 'kl/beta': 0.006071132142096758, 'kl/avg_steps': 0.71875, 'epoch': 0.53}
|
|||
|
|
53%|████████████████████████████████████████████████████████████ | 359/681 [21:33<13:36, 2.54s/it]
53%|████████████████████████████████████████████████████████████▎ | 360/681 [21:35<13:43, 2.56s/it]
{'loss': 0.6196, 'grad_norm': 53.97544860839844, 'learning_rate': 2.7051020734928443e-07, 'rewards/chosen': -0.7889617681503296, 'rewards/rejected': -2.149751901626587, 'rewards/accuracies': 0.875, 'rewards/margins': 1.3607900142669678, 'logps/chosen': -168.40216064453125, 'logps/rejected': -436.15087890625, 'logps/ref_chosen': -37.17042922973633, 'logps/ref_rejected': -76.79757690429688, 'logits/chosen': -2.0680994987487793, 'logits/rejected': -1.957366704940796, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.005986965261399746, 'epsilon_dpo/loss_margin_mean': 228.12159729003906, 'epsilon_dpo/beta_margin_mean': 1.3607901334762573, 'epsilon_dpo/beta_margin_std': 0.9962857365608215, 'epsilon_dpo/beta_margin_grad_mean': -0.2441015988588333, 'epsilon_dpo/beta_margin_grad_std': 0.1673990935087204, 'kl/beta': 0.006027807481586933, 'kl/avg_steps': 0.6875, 'epoch': 0.53}
|
|||
|
|
53%|████████████████████████████████████████████████████████████▎ | 360/681 [21:35<13:43, 2.56s/it]
53%|████████████████████████████████████████████████████████████▍ | 361/681 [21:38<13:43, 2.57s/it]
{'loss': 0.6539, 'grad_norm': 63.17055892944336, 'learning_rate': 2.6923093854861593e-07, 'rewards/chosen': -0.7777143120765686, 'rewards/rejected': -2.148881435394287, 'rewards/accuracies': 0.875, 'rewards/margins': 1.3711671829223633, 'logps/chosen': -164.0542755126953, 'logps/rejected': -457.8388671875, 'logps/ref_chosen': -33.520301818847656, 'logps/ref_rejected': -95.78604125976562, 'logits/chosen': -2.1590943336486816, 'logits/rejected': -2.121795177459717, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.005942343734204769, 'epsilon_dpo/loss_margin_mean': 231.51885986328125, 'epsilon_dpo/beta_margin_mean': 1.3711673021316528, 'epsilon_dpo/beta_margin_std': 1.1313101053237915, 'epsilon_dpo/beta_margin_grad_mean': -0.25044846534729004, 'epsilon_dpo/beta_margin_grad_std': 0.18136946856975555, 'kl/beta': 0.005986649077385664, 'kl/avg_steps': 0.75, 'epoch': 0.53}
|
|||
|
|
53%|████████████████████████████████████████████████████████████▍ | 361/681 [21:38<13:43, 2.57s/it]
53%|████████████████████████████████████████████████████████████▌ | 362/681 [21:40<13:23, 2.52s/it]
{'loss': 0.5872, 'grad_norm': 57.27875900268555, 'learning_rate': 2.679511629948319e-07, 'rewards/chosen': -0.8333665728569031, 'rewards/rejected': -2.2787985801696777, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.4454319477081299, 'logps/chosen': -185.8004913330078, 'logps/rejected': -497.8592834472656, 'logps/ref_chosen': -44.4540901184082, 'logps/ref_rejected': -110.75244140625, 'logits/chosen': -2.2858972549438477, 'logits/rejected': -2.1964077949523926, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.005890679080039263, 'epsilon_dpo/loss_margin_mean': 245.76043701171875, 'epsilon_dpo/beta_margin_mean': 1.4454318284988403, 'epsilon_dpo/beta_margin_std': 1.0421142578125, 'epsilon_dpo/beta_margin_grad_mean': -0.23398736119270325, 'epsilon_dpo/beta_margin_grad_std': 0.1614319384098053, 'kl/beta': 0.005942083429545164, 'kl/avg_steps': 0.875, 'epoch': 0.53}
|
|||
|
|
53%|████████████████████████████████████████████████████████████▌ | 362/681 [21:40<13:23, 2.52s/it]
53%|████████████████████████████████████████████████████████████▊ | 363/681 [21:42<13:05, 2.47s/it]
{'loss': 0.4544, 'grad_norm': 44.063777923583984, 'learning_rate': 2.6667091441120816e-07, 'rewards/chosen': -0.5620150566101074, 'rewards/rejected': -2.316688060760498, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.7546730041503906, 'logps/chosen': -119.686767578125, 'logps/rejected': -476.1310119628906, 'logps/ref_chosen': -23.62018585205078, 'logps/ref_rejected': -79.18755340576172, 'logits/chosen': -2.20405650138855, 'logits/rejected': -2.084122657775879, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.005839582998305559, 'epsilon_dpo/loss_margin_mean': 300.8768615722656, 'epsilon_dpo/beta_margin_mean': 1.7546730041503906, 'epsilon_dpo/beta_margin_std': 1.0252107381820679, 'epsilon_dpo/beta_margin_grad_mean': -0.18884162604808807, 'epsilon_dpo/beta_margin_grad_std': 0.14109890162944794, 'kl/beta': 0.00589054124429822, 'kl/avg_steps': 0.875, 'epoch': 0.53}
|
|||
|
|
53%|████████████████████████████████████████████████████████████▊ | 363/681 [21:43<13:05, 2.47s/it]
53%|████████████████████████████████████████████████████████████▉ | 364/681 [21:45<13:09, 2.49s/it]
{'loss': 0.5787, 'grad_norm': 54.10791015625, 'learning_rate': 2.6539022653348575e-07, 'rewards/chosen': -0.6957234144210815, 'rewards/rejected': -2.118544101715088, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.4228205680847168, 'logps/chosen': -151.59072875976562, 'logps/rejected': -464.345703125, 'logps/ref_chosen': -31.6948184967041, 'logps/ref_rejected': -98.25422668457031, 'logits/chosen': -2.1630282402038574, 'logits/rejected': -2.1442086696624756, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.005788929760456085, 'epsilon_dpo/loss_margin_mean': 246.19557189941406, 'epsilon_dpo/beta_margin_mean': 1.4228205680847168, 'epsilon_dpo/beta_margin_std': 0.979676365852356, 'epsilon_dpo/beta_margin_grad_mean': -0.23159046471118927, 'epsilon_dpo/beta_margin_grad_std': 0.15345177054405212, 'kl/beta': 0.005839446093887091, 'kl/avg_steps': 0.875, 'epoch': 0.53}
|
|||
|
|
53%|████████████████████████████████████████████████████████████▉ | 364/681 [21:45<13:09, 2.49s/it]
54%|█████████████████████████████████████████████████████████████ | 365/681 [21:47<12:53, 2.45s/it]
{'loss': 0.5397, 'grad_norm': 45.98625564575195, 'learning_rate': 2.641091331089811e-07, 'rewards/chosen': -0.5555899739265442, 'rewards/rejected': -2.0330123901367188, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.4774224758148193, 'logps/chosen': -125.05105590820312, 'logps/rejected': -452.42401123046875, 'logps/ref_chosen': -28.370468139648438, 'logps/ref_rejected': -97.9366226196289, 'logits/chosen': -2.2106542587280273, 'logits/rejected': -2.1826648712158203, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.005736907012760639, 'epsilon_dpo/loss_margin_mean': 257.8067932128906, 'epsilon_dpo/beta_margin_mean': 1.4774224758148193, 'epsilon_dpo/beta_margin_std': 0.9550539255142212, 'epsilon_dpo/beta_margin_grad_mean': -0.22249744832515717, 'epsilon_dpo/beta_margin_grad_std': 0.13856177031993866, 'kl/beta': 0.005788794253021479, 'kl/avg_steps': 0.90625, 'epoch': 0.54}
|
|||
|
|
54%|█████████████████████████████████████████████████████████████ | 365/681 [21:47<12:53, 2.45s/it]
54%|█████████████████████████████████████████████████████████████▎ | 366/681 [21:50<12:50, 2.45s/it]
{'loss': 0.605, 'grad_norm': 45.932926177978516, 'learning_rate': 2.6282766789569736e-07, 'rewards/chosen': -0.6359978914260864, 'rewards/rejected': -2.022883176803589, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.3868852853775024, 'logps/chosen': -141.24298095703125, 'logps/rejected': -447.4891357421875, 'logps/ref_chosen': -29.695941925048828, 'logps/ref_rejected': -91.68377685546875, 'logits/chosen': -2.1703920364379883, 'logits/rejected': -2.1496224403381348, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.005688969045877457, 'epsilon_dpo/loss_margin_mean': 244.25833129882812, 'epsilon_dpo/beta_margin_mean': 1.3868852853775024, 'epsilon_dpo/beta_margin_std': 1.0170059204101562, 'epsilon_dpo/beta_margin_grad_mean': -0.241429403424263, 'epsilon_dpo/beta_margin_grad_std': 0.15970946848392487, 'kl/beta': 0.0057368045672774315, 'kl/avg_steps': 0.84375, 'epoch': 0.54}
|
|||
|
|
54%|█████████████████████████████████████████████████████████████▎ | 366/681 [21:50<12:50, 2.45s/it]
54%|█████████████████████████████████████████████████████████████▍ | 367/681 [21:52<13:03, 2.49s/it]
{'loss': 0.5899, 'grad_norm': 42.18586730957031, 'learning_rate': 2.615458646614349e-07, 'rewards/chosen': -0.624735951423645, 'rewards/rejected': -1.9736195802688599, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.3488836288452148, 'logps/chosen': -153.87405395507812, 'logps/rejected': -432.596435546875, 'logps/ref_chosen': -43.07976531982422, 'logps/ref_rejected': -82.26808166503906, 'logits/chosen': -2.2993650436401367, 'logits/rejected': -2.1403145790100098, 'kl/p_epsilon_steps': 0.984375, 'kl/n_epsilon_steps': 0.015625, 'epsilon_dpo/beta': 0.005634258035570383, 'epsilon_dpo/loss_margin_mean': 239.53407287597656, 'epsilon_dpo/beta_margin_mean': 1.3488836288452148, 'epsilon_dpo/beta_margin_std': 0.8744153380393982, 'epsilon_dpo/beta_margin_grad_mean': -0.23779228329658508, 'epsilon_dpo/beta_margin_grad_std': 0.14979609847068787, 'kl/beta': 0.005688805133104324, 'kl/avg_steps': 0.96875, 'epoch': 0.54}
|
|||
|
|
54%|█████████████████████████████████████████████████████████████▍ | 367/681 [21:52<13:03, 2.49s/it]
54%|█████████████████████████████████████████████████████████████▌ | 368/681 [21:55<13:20, 2.56s/it]
{'loss': 0.5089, 'grad_norm': 42.2542610168457, 'learning_rate': 2.6026375718290083e-07, 'rewards/chosen': -0.633552074432373, 'rewards/rejected': -2.111250400543213, 'rewards/accuracies': 0.984375, 'rewards/margins': 1.4776982069015503, 'logps/chosen': -142.2340545654297, 'logps/rejected': -482.7167663574219, 'logps/ref_chosen': -28.834022521972656, 'logps/ref_rejected': -104.38568115234375, 'logits/chosen': -2.189484119415283, 'logits/rejected': -2.1851706504821777, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.005581961013376713, 'epsilon_dpo/loss_margin_mean': 264.9310607910156, 'epsilon_dpo/beta_margin_mean': 1.4776982069015503, 'epsilon_dpo/beta_margin_std': 0.806984007358551, 'epsilon_dpo/beta_margin_grad_mean': -0.2129683494567871, 'epsilon_dpo/beta_margin_grad_std': 0.1246427670121193, 'kl/beta': 0.005634223576635122, 'kl/avg_steps': 0.9375, 'epoch': 0.54}
|
|||
|
|
54%|█████████████████████████████████████████████████████████████▌ | 368/681 [21:55<13:20, 2.56s/it]
54%|█████████████████████████████████████████████████████████████▊ | 369/681 [21:58<13:15, 2.55s/it]
{'loss': 0.5458, 'grad_norm': 39.924076080322266, 'learning_rate': 2.589813792448196e-07, 'rewards/chosen': -0.640053391456604, 'rewards/rejected': -2.0713844299316406, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.431330919265747, 'logps/chosen': -161.64993286132812, 'logps/rejected': -472.6080017089844, 'logps/ref_chosen': -46.21360778808594, 'logps/ref_rejected': -98.12519836425781, 'logits/chosen': -2.307586193084717, 'logits/rejected': -2.1981377601623535, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.0055318609811365604, 'epsilon_dpo/loss_margin_mean': 259.0464782714844, 'epsilon_dpo/beta_margin_mean': 1.431330919265747, 'epsilon_dpo/beta_margin_std': 0.8688164353370667, 'epsilon_dpo/beta_margin_grad_mean': -0.223898246884346, 'epsilon_dpo/beta_margin_grad_std': 0.13859553635120392, 'kl/beta': 0.005581893492490053, 'kl/avg_steps': 0.90625, 'epoch': 0.54}
|
|||
|
|
54%|█████████████████████████████████████████████████████████████▊ | 369/681 [21:58<13:15, 2.55s/it]
54%|█████████████████████████████████████████████████████████████▉ | 370/681 [22:00<13:17, 2.56s/it]
{'loss': 0.6656, 'grad_norm': 44.1972770690918, 'learning_rate': 2.5769876463904263e-07, 'rewards/chosen': -0.7640899419784546, 'rewards/rejected': -1.997938871383667, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.2338488101959229, 'logps/chosen': -190.09152221679688, 'logps/rejected': -465.984130859375, 'logps/ref_chosen': -51.13551330566406, 'logps/ref_rejected': -101.64096069335938, 'logits/chosen': -2.3571860790252686, 'logits/rejected': -2.2904090881347656, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.005487365182489157, 'epsilon_dpo/loss_margin_mean': 225.38717651367188, 'epsilon_dpo/beta_margin_mean': 1.2338489294052124, 'epsilon_dpo/beta_margin_std': 0.9565902352333069, 'epsilon_dpo/beta_margin_grad_mean': -0.26117002964019775, 'epsilon_dpo/beta_margin_grad_std': 0.16114374995231628, 'kl/beta': 0.005531761795282364, 'kl/avg_steps': 0.8125, 'epoch': 0.54}
|
|||
|
|
54%|█████████████████████████████████████████████████████████████▉ | 370/681 [22:00<13:17, 2.56s/it]
54%|██████████████████████████████████████████████████████████████ | 371/681 [22:03<13:15, 2.57s/it]
{'loss': 0.65, 'grad_norm': 59.9031982421875, 'learning_rate': 2.5641594716365744e-07, 'rewards/chosen': -0.7477129697799683, 'rewards/rejected': -2.094331741333008, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.34661865234375, 'logps/chosen': -181.4659881591797, 'logps/rejected': -491.27606201171875, 'logps/ref_chosen': -44.41009521484375, 'logps/ref_rejected': -106.13936614990234, 'logits/chosen': -2.2778027057647705, 'logits/rejected': -2.235936164855957, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.005443139001727104, 'epsilon_dpo/loss_margin_mean': 248.08079528808594, 'epsilon_dpo/beta_margin_mean': 1.3466187715530396, 'epsilon_dpo/beta_margin_std': 1.0779163837432861, 'epsilon_dpo/beta_margin_grad_mean': -0.2511758804321289, 'epsilon_dpo/beta_margin_grad_std': 0.17598623037338257, 'kl/beta': 0.005487178452312946, 'kl/avg_steps': 0.8125, 'epoch': 0.54}
|
|||
|
|
54%|██████████████████████████████████████████████████████████████ | 371/681 [22:03<13:15, 2.57s/it]
55%|██████████████████████████████████████████████████████████████▎ | 372/681 [22:05<13:12, 2.57s/it]
{'loss': 0.5632, 'grad_norm': 42.31486511230469, 'learning_rate': 2.551329606220976e-07, 'rewards/chosen': -0.7031924724578857, 'rewards/rejected': -2.2069644927978516, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.5037720203399658, 'logps/chosen': -168.07699584960938, 'logps/rejected': -494.7847595214844, 'logps/ref_chosen': -37.96990203857422, 'logps/ref_rejected': -85.50196838378906, 'logits/chosen': -2.188450336456299, 'logits/rejected': -1.9817776679992676, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.005397569388151169, 'epsilon_dpo/loss_margin_mean': 279.17572021484375, 'epsilon_dpo/beta_margin_mean': 1.5037719011306763, 'epsilon_dpo/beta_margin_std': 1.0139695405960083, 'epsilon_dpo/beta_margin_grad_mean': -0.2212544083595276, 'epsilon_dpo/beta_margin_grad_std': 0.16570396721363068, 'kl/beta': 0.00544295459985733, 'kl/avg_steps': 0.84375, 'epoch': 0.55}
|
|||
|
|
55%|██████████████████████████████████████████████████████████████▎ | 372/681 [22:05<13:12, 2.57s/it]
55%|██████████████████████████████████████████████████████████████▍ | 373/681 [22:08<13:06, 2.56s/it]
{'loss': 0.5204, 'grad_norm': 46.6475944519043, 'learning_rate': 2.538498388222517e-07, 'rewards/chosen': -0.6824929118156433, 'rewards/rejected': -2.2047948837280273, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.5223019123077393, 'logps/chosen': -159.17428588867188, 'logps/rejected': -505.668701171875, 'logps/ref_chosen': -31.804325103759766, 'logps/ref_rejected': -93.370361328125, 'logits/chosen': -2.178375005722046, 'logits/rejected': -2.014909267425537, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.005349034443497658, 'epsilon_dpo/loss_margin_mean': 284.92840576171875, 'epsilon_dpo/beta_margin_mean': 1.5223017930984497, 'epsilon_dpo/beta_margin_std': 0.9210037589073181, 'epsilon_dpo/beta_margin_grad_mean': -0.2134443074464798, 'epsilon_dpo/beta_margin_grad_std': 0.14159946143627167, 'kl/beta': 0.005397413857281208, 'kl/avg_steps': 0.90625, 'epoch': 0.55}
|
|||
|
|
55%|██████████████████████████████████████████████████████████████▍ | 373/681 [22:08<13:06, 2.56s/it]
55%|██████████████████████████████████████████████████████████████▌ | 374/681 [22:11<13:08, 2.57s/it]
{'loss': 0.6052, 'grad_norm': 65.01420593261719, 'learning_rate': 2.525666155755725e-07, 'rewards/chosen': -0.7352691888809204, 'rewards/rejected': -2.06484317779541, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.3295738697052002, 'logps/chosen': -186.239501953125, 'logps/rejected': -489.414794921875, 'logps/ref_chosen': -47.937625885009766, 'logps/ref_rejected': -99.9871826171875, 'logits/chosen': -2.1958394050598145, 'logits/rejected': -2.096358060836792, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.005304337944835424, 'epsilon_dpo/loss_margin_mean': 251.12574768066406, 'epsilon_dpo/beta_margin_mean': 1.3295738697052002, 'epsilon_dpo/beta_margin_std': 0.9483336806297302, 'epsilon_dpo/beta_margin_grad_mean': -0.24420872330665588, 'epsilon_dpo/beta_margin_grad_std': 0.14603427052497864, 'kl/beta': 0.00534893898293376, 'kl/avg_steps': 0.84375, 'epoch': 0.55}
|
|||
|
|
55%|██████████████████████████████████████████████████████████████▌ | 374/681 [22:11<13:08, 2.57s/it]
55%|██████████████████████████████████████████████████████████████▊ | 375/681 [22:13<12:58, 2.54s/it]
{'loss': 0.5067, 'grad_norm': 44.36325454711914, 'learning_rate': 2.512833246961859e-07, 'rewards/chosen': -0.6010797023773193, 'rewards/rejected': -2.221287727355957, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.6202080249786377, 'logps/chosen': -151.54449462890625, 'logps/rejected': -518.91259765625, 'logps/ref_chosen': -37.31241989135742, 'logps/ref_rejected': -96.08183288574219, 'logits/chosen': -2.20353364944458, 'logits/rejected': -2.1390933990478516, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.005254983436316252, 'epsilon_dpo/loss_margin_mean': 308.59869384765625, 'epsilon_dpo/beta_margin_mean': 1.6202079057693481, 'epsilon_dpo/beta_margin_std': 1.0169481039047241, 'epsilon_dpo/beta_margin_grad_mean': -0.20512135326862335, 'epsilon_dpo/beta_margin_grad_std': 0.1477726697921753, 'kl/beta': 0.0053041847422719, 'kl/avg_steps': 0.9375, 'epoch': 0.55}
|
|||
|
|
55%|██████████████████████████████████████████████████████████████▊ | 375/681 [22:13<12:58, 2.54s/it]
55%|██████████████████████████████████████████████████████████████▉ | 376/681 [22:16<12:52, 2.53s/it]
{'loss': 0.5114, 'grad_norm': 37.207130432128906, 'learning_rate': 2.5e-07, 'rewards/chosen': -0.605557918548584, 'rewards/rejected': -2.264993190765381, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.6594352722167969, 'logps/chosen': -162.23599243164062, 'logps/rejected': -547.9850463867188, 'logps/ref_chosen': -46.223838806152344, 'logps/ref_rejected': -112.95925903320312, 'logits/chosen': -2.2137274742126465, 'logits/rejected': -2.0475354194641113, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.005209460388869047, 'epsilon_dpo/loss_margin_mean': 319.01361083984375, 'epsilon_dpo/beta_margin_mean': 1.6594352722167969, 'epsilon_dpo/beta_margin_std': 1.0614945888519287, 'epsilon_dpo/beta_margin_grad_mean': -0.20545507967472076, 'epsilon_dpo/beta_margin_grad_std': 0.1631988137960434, 'kl/beta': 0.005254920106381178, 'kl/avg_steps': 0.875, 'epoch': 0.55}
|
|||
|
|
55%|██████████████████████████████████████████████████████████████▉ | 376/681 [22:16<12:52, 2.53s/it]
55%|███████████████████████████████████████████████████████████████ | 377/681 [22:18<12:45, 2.52s/it]
{'loss': 0.5956, 'grad_norm': 44.168060302734375, 'learning_rate': 2.487166753038141e-07, 'rewards/chosen': -0.7789356708526611, 'rewards/rejected': -2.1247482299804688, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.3458126783370972, 'logps/chosen': -185.23043823242188, 'logps/rejected': -517.095947265625, 'logps/ref_chosen': -34.36379623413086, 'logps/ref_rejected': -105.25419616699219, 'logits/chosen': -2.0519776344299316, 'logits/rejected': -2.01825213432312, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.005162645131349564, 'epsilon_dpo/loss_margin_mean': 260.9751281738281, 'epsilon_dpo/beta_margin_mean': 1.3458126783370972, 'epsilon_dpo/beta_margin_std': 0.9675434827804565, 'epsilon_dpo/beta_margin_grad_mean': -0.24356918036937714, 'epsilon_dpo/beta_margin_grad_std': 0.13881158828735352, 'kl/beta': 0.0052093383856117725, 'kl/avg_steps': 0.90625, 'epoch': 0.55}
|
|||
|
|
55%|███████████████████████████████████████████████████████████████ | 377/681 [22:18<12:45, 2.52s/it]
56%|███████████████████████████████████████████████████████████████▎ | 378/681 [22:20<12:15, 2.43s/it]
{'loss': 0.5897, 'grad_norm': 52.31544876098633, 'learning_rate': 2.4743338442442754e-07, 'rewards/chosen': -0.6769013404846191, 'rewards/rejected': -2.1067166328430176, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.4298152923583984, 'logps/chosen': -163.18124389648438, 'logps/rejected': -506.7203674316406, 'logps/ref_chosen': -31.191333770751953, 'logps/ref_rejected': -94.8112564086914, 'logits/chosen': -2.059394359588623, 'logits/rejected': -1.9235854148864746, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.005117891822010279, 'epsilon_dpo/loss_margin_mean': 279.919189453125, 'epsilon_dpo/beta_margin_mean': 1.4298152923583984, 'epsilon_dpo/beta_margin_std': 0.9799423217773438, 'epsilon_dpo/beta_margin_grad_mean': -0.22632895410060883, 'epsilon_dpo/beta_margin_grad_std': 0.1629311591386795, 'kl/beta': 0.005162552464753389, 'kl/avg_steps': 0.875, 'epoch': 0.56}
|
|||
|
|
56%|███████████████████████████████████████████████████████████████▎ | 378/681 [22:20<12:15, 2.43s/it]
56%|███████████████████████████████████████████████████████████████▍ | 379/681 [22:23<12:13, 2.43s/it]
{'loss': 0.6029, 'grad_norm': 51.661922454833984, 'learning_rate': 2.461501611777483e-07, 'rewards/chosen': -0.8247177004814148, 'rewards/rejected': -2.2538905143737793, 'rewards/accuracies': 0.875, 'rewards/margins': 1.4291728734970093, 'logps/chosen': -197.74441528320312, 'logps/rejected': -564.896728515625, 'logps/ref_chosen': -35.68910217285156, 'logps/ref_rejected': -120.69367980957031, 'logits/chosen': -2.0814695358276367, 'logits/rejected': -2.1467087268829346, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.005076698027551174, 'epsilon_dpo/loss_margin_mean': 282.147705078125, 'epsilon_dpo/beta_margin_mean': 1.4291728734970093, 'epsilon_dpo/beta_margin_std': 1.0851447582244873, 'epsilon_dpo/beta_margin_grad_mean': -0.23987461626529694, 'epsilon_dpo/beta_margin_grad_std': 0.16330820322036743, 'kl/beta': 0.005117772147059441, 'kl/avg_steps': 0.8125, 'epoch': 0.56}
|
|||
|
|
56%|███████████████████████████████████████████████████████████████▍ | 379/681 [22:23<12:13, 2.43s/it]
56%|███████████████████████████████████████████████████████████████▌ | 380/681 [22:25<12:13, 2.44s/it]
{'loss': 0.5118, 'grad_norm': 39.429542541503906, 'learning_rate': 2.4486703937790243e-07, 'rewards/chosen': -0.7753435373306274, 'rewards/rejected': -2.4232218265533447, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.6478781700134277, 'logps/chosen': -184.11827087402344, 'logps/rejected': -593.0174560546875, 'logps/ref_chosen': -30.20886993408203, 'logps/ref_rejected': -111.17587280273438, 'logits/chosen': -2.105628728866577, 'logits/rejected': -2.135561466217041, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.005032609216868877, 'epsilon_dpo/loss_margin_mean': 327.93218994140625, 'epsilon_dpo/beta_margin_mean': 1.6478782892227173, 'epsilon_dpo/beta_margin_std': 1.1102581024169922, 'epsilon_dpo/beta_margin_grad_mean': -0.20714667439460754, 'epsilon_dpo/beta_margin_grad_std': 0.1527094542980194, 'kl/beta': 0.005076525267213583, 'kl/avg_steps': 0.875, 'epoch': 0.56}
|
|||
|
|
56%|███████████████████████████████████████████████████████████████▌ | 380/681 [22:25<12:13, 2.44s/it]
56%|███████████████████████████████████████████████████████████████▊ | 381/681 [22:28<12:29, 2.50s/it]
{'loss': 0.64, 'grad_norm': 51.32357406616211, 'learning_rate': 2.435840528363426e-07, 'rewards/chosen': -0.8029674291610718, 'rewards/rejected': -2.109452247619629, 'rewards/accuracies': 0.859375, 'rewards/margins': 1.3064846992492676, 'logps/chosen': -195.97369384765625, 'logps/rejected': -506.63018798828125, 'logps/ref_chosen': -35.8305778503418, 'logps/ref_rejected': -84.19041442871094, 'logits/chosen': -2.112166404724121, 'logits/rejected': -1.9345085620880127, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.004996819421648979, 'epsilon_dpo/loss_margin_mean': 262.2966613769531, 'epsilon_dpo/beta_margin_mean': 1.3064846992492676, 'epsilon_dpo/beta_margin_std': 1.0072511434555054, 'epsilon_dpo/beta_margin_grad_mean': -0.2540000379085541, 'epsilon_dpo/beta_margin_grad_std': 0.1617269366979599, 'kl/beta': 0.005032490938901901, 'kl/avg_steps': 0.71875, 'epoch': 0.56}
|
|||
|
|
56%|███████████████████████████████████████████████████████████████▊ | 381/681 [22:28<12:29, 2.50s/it]
56%|███████████████████████████████████████████████████████████████▉ | 382/681 [22:30<12:32, 2.52s/it]
{'loss': 0.5931, 'grad_norm': 47.54762268066406, 'learning_rate': 2.4230123536095745e-07, 'rewards/chosen': -0.839522123336792, 'rewards/rejected': -2.2582650184631348, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.4187428951263428, 'logps/chosen': -211.81280517578125, 'logps/rejected': -572.606201171875, 'logps/ref_chosen': -42.912322998046875, 'logps/ref_rejected': -116.84707641601562, 'logits/chosen': -2.1727161407470703, 'logits/rejected': -2.1786251068115234, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.004958038218319416, 'epsilon_dpo/loss_margin_mean': 286.858642578125, 'epsilon_dpo/beta_margin_mean': 1.4187428951263428, 'epsilon_dpo/beta_margin_std': 1.0073864459991455, 'epsilon_dpo/beta_margin_grad_mean': -0.23661740124225616, 'epsilon_dpo/beta_margin_grad_std': 0.16259969770908356, 'kl/beta': 0.004996578209102154, 'kl/avg_steps': 0.78125, 'epoch': 0.56}
|
|||
|
|
56%|███████████████████████████████████████████████████████████████▉ | 382/681 [22:30<12:32, 2.52s/it]
56%|████████████████████████████████████████████████████████████████ | 383/681 [22:33<12:42, 2.56s/it]
{'loss': 0.6678, 'grad_norm': 56.97015380859375, 'learning_rate': 2.4101862075518037e-07, 'rewards/chosen': -0.8556368947029114, 'rewards/rejected': -2.160849094390869, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.3052120208740234, 'logps/chosen': -209.27877807617188, 'logps/rejected': -539.654052734375, 'logps/ref_chosen': -35.82965087890625, 'logps/ref_rejected': -100.10868072509766, 'logits/chosen': -2.0988528728485107, 'logits/rejected': -2.077052354812622, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.00491960346698761, 'epsilon_dpo/loss_margin_mean': 266.0962219238281, 'epsilon_dpo/beta_margin_mean': 1.305212140083313, 'epsilon_dpo/beta_margin_std': 1.0758684873580933, 'epsilon_dpo/beta_margin_grad_mean': -0.2571463882923126, 'epsilon_dpo/beta_margin_grad_std': 0.1763610541820526, 'kl/beta': 0.004957844968885183, 'kl/avg_steps': 0.78125, 'epoch': 0.56}
|
|||
|
|
56%|████████████████████████████████████████████████████████████████ | 383/681 [22:33<12:42, 2.56s/it]
56%|████████████████████████████████████████████████████████████████▎ | 384/681 [22:35<12:25, 2.51s/it]
{'loss': 0.6509, 'grad_norm': 49.46163558959961, 'learning_rate': 2.397362428170992e-07, 'rewards/chosen': -0.8223646879196167, 'rewards/rejected': -1.998728632926941, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.1763639450073242, 'logps/chosen': -204.53836059570312, 'logps/rejected': -497.9023132324219, 'logps/ref_chosen': -36.32765579223633, 'logps/ref_rejected': -88.09075927734375, 'logits/chosen': -2.1388659477233887, 'logits/rejected': -2.1503121852874756, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.004878391977399588, 'epsilon_dpo/loss_margin_mean': 241.60084533691406, 'epsilon_dpo/beta_margin_mean': 1.1763639450073242, 'epsilon_dpo/beta_margin_std': 0.7963204979896545, 'epsilon_dpo/beta_margin_grad_mean': -0.261491596698761, 'epsilon_dpo/beta_margin_grad_std': 0.14361578226089478, 'kl/beta': 0.004919412080198526, 'kl/avg_steps': 0.84375, 'epoch': 0.56}
|
|||
|
|
56%|████████████████████████████████████████████████████████████████▎ | 384/681 [22:35<12:25, 2.51s/it]
57%|████████████████████████████████████████████████████████████████▍ | 385/681 [22:38<12:09, 2.47s/it]
{'loss': 0.595, 'grad_norm': 42.43720245361328, 'learning_rate': 2.3845413533856514e-07, 'rewards/chosen': -0.7839208841323853, 'rewards/rejected': -2.1058101654052734, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.3218892812728882, 'logps/chosen': -205.151123046875, 'logps/rejected': -520.9268188476562, 'logps/ref_chosen': -43.28904724121094, 'logps/ref_rejected': -85.31612396240234, 'logits/chosen': -2.1851940155029297, 'logits/rejected': -1.9630916118621826, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.004834526218473911, 'epsilon_dpo/loss_margin_mean': 273.7486267089844, 'epsilon_dpo/beta_margin_mean': 1.3218892812728882, 'epsilon_dpo/beta_margin_std': 0.876196563243866, 'epsilon_dpo/beta_margin_grad_mean': -0.24184301495552063, 'epsilon_dpo/beta_margin_grad_std': 0.1403016448020935, 'kl/beta': 0.004878251813352108, 'kl/avg_steps': 0.90625, 'epoch': 0.57}
|
|||
|
|
57%|████████████████████████████████████████████████████████████████▍ | 385/681 [22:38<12:09, 2.47s/it]
57%|████████████████████████████████████████████████████████████████▌ | 386/681 [22:40<12:12, 2.48s/it]
{'loss': 0.5412, 'grad_norm': 44.11907196044922, 'learning_rate': 2.3717233210430254e-07, 'rewards/chosen': -0.7281551957130432, 'rewards/rejected': -2.1762986183166504, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.448143482208252, 'logps/chosen': -186.66197204589844, 'logps/rejected': -553.1368408203125, 'logps/ref_chosen': -34.78173828125, 'logps/ref_rejected': -98.69821166992188, 'logits/chosen': -2.1383421421051025, 'logits/rejected': -2.0959932804107666, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.004791106563061476, 'epsilon_dpo/loss_margin_mean': 302.5584411621094, 'epsilon_dpo/beta_margin_mean': 1.448143482208252, 'epsilon_dpo/beta_margin_std': 0.916283369064331, 'epsilon_dpo/beta_margin_grad_mean': -0.22500666975975037, 'epsilon_dpo/beta_margin_grad_std': 0.1310817301273346, 'kl/beta': 0.004834439605474472, 'kl/avg_steps': 0.90625, 'epoch': 0.57}
|
|||
|
|
57%|████████████████████████████████████████████████████████████████▌ | 386/681 [22:40<12:12, 2.48s/it]
57%|████████████████████████████████████████████████████████████████▊ | 387/681 [22:43<12:15, 2.50s/it]
{'loss': 0.6613, 'grad_norm': 52.421470642089844, 'learning_rate': 2.3589086689101889e-07, 'rewards/chosen': -0.8966184854507446, 'rewards/rejected': -2.0666656494140625, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.1700471639633179, 'logps/chosen': -236.2892608642578, 'logps/rejected': -532.7896728515625, 'logps/ref_chosen': -47.62559509277344, 'logps/ref_rejected': -97.41331481933594, 'logits/chosen': -2.315922737121582, 'logits/rejected': -2.0701866149902344, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.004748077597469091, 'epsilon_dpo/loss_margin_mean': 246.71267700195312, 'epsilon_dpo/beta_margin_mean': 1.1700471639633179, 'epsilon_dpo/beta_margin_std': 0.8362559080123901, 'epsilon_dpo/beta_margin_grad_mean': -0.2660858631134033, 'epsilon_dpo/beta_margin_grad_std': 0.1432296484708786, 'kl/beta': 0.0047910213470458984, 'kl/avg_steps': 0.90625, 'epoch': 0.57}
|
|||
|
|
57%|████████████████████████████████████████████████████████████████▊ | 387/681 [22:43<12:15, 2.50s/it]
57%|████████████████████████████████████████████████████████████████▉ | 388/681 [22:45<12:05, 2.48s/it]
{'loss': 0.5907, 'grad_norm': 50.50879669189453, 'learning_rate': 2.3460977346651428e-07, 'rewards/chosen': -0.767310619354248, 'rewards/rejected': -2.153362512588501, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.386051893234253, 'logps/chosen': -198.59793090820312, 'logps/rejected': -568.3635864257812, 'logps/ref_chosen': -35.743316650390625, 'logps/ref_rejected': -110.60121154785156, 'logits/chosen': -2.154555320739746, 'logits/rejected': -2.115201473236084, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.004706918261945248, 'epsilon_dpo/loss_margin_mean': 294.90777587890625, 'epsilon_dpo/beta_margin_mean': 1.386051893234253, 'epsilon_dpo/beta_margin_std': 0.9813371300697327, 'epsilon_dpo/beta_margin_grad_mean': -0.23977012932300568, 'epsilon_dpo/beta_margin_grad_std': 0.14799103140830994, 'kl/beta': 0.004747992381453514, 'kl/avg_steps': 0.875, 'epoch': 0.57}
|
|||
|
|
57%|████████████████████████████████████████████████████████████████▉ | 388/681 [22:45<12:05, 2.48s/it]
57%|█████████████████████████████████████████████████████████████████ | 389/681 [22:48<11:59, 2.46s/it]
{'loss': 0.7075, 'grad_norm': 57.76890563964844, 'learning_rate': 2.3332908558879177e-07, 'rewards/chosen': -0.7245763540267944, 'rewards/rejected': -1.9424829483032227, 'rewards/accuracies': 0.859375, 'rewards/margins': 1.2179067134857178, 'logps/chosen': -195.3277587890625, 'logps/rejected': -498.21917724609375, 'logps/ref_chosen': -40.81889724731445, 'logps/ref_rejected': -82.08807373046875, 'logits/chosen': -2.1483242511749268, 'logits/rejected': -2.025583267211914, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.004673445131629705, 'epsilon_dpo/loss_margin_mean': 261.6222839355469, 'epsilon_dpo/beta_margin_mean': 1.2179065942764282, 'epsilon_dpo/beta_margin_std': 1.0329232215881348, 'epsilon_dpo/beta_margin_grad_mean': -0.26743534207344055, 'epsilon_dpo/beta_margin_grad_std': 0.18217839300632477, 'kl/beta': 0.004706807900220156, 'kl/avg_steps': 0.71875, 'epoch': 0.57}
|
|||
|
|
57%|█████████████████████████████████████████████████████████████████ | 389/681 [22:48<11:59, 2.46s/it]
57%|█████████████████████████████████████████████████████████████████▎ | 390/681 [22:50<12:03, 2.48s/it]
{'loss': 0.5862, 'grad_norm': 47.92219161987305, 'learning_rate': 2.320488370051681e-07, 'rewards/chosen': -0.6144026517868042, 'rewards/rejected': -1.959489107131958, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.3450864553451538, 'logps/chosen': -161.92718505859375, 'logps/rejected': -516.030517578125, 'logps/ref_chosen': -29.450889587402344, 'logps/ref_rejected': -92.78984069824219, 'logits/chosen': -2.114452838897705, 'logits/rejected': -1.998061180114746, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.004632791969925165, 'epsilon_dpo/loss_margin_mean': 290.7643737792969, 'epsilon_dpo/beta_margin_mean': 1.3450864553451538, 'epsilon_dpo/beta_margin_std': 0.9246547222137451, 'epsilon_dpo/beta_margin_grad_mean': -0.24001896381378174, 'epsilon_dpo/beta_margin_grad_std': 0.13718777894973755, 'kl/beta': 0.004673219285905361, 'kl/avg_steps': 0.875, 'epoch': 0.57}
|
|||
|
|
57%|█████████████████████████████████████████████████████████████████▎ | 390/681 [22:50<12:03, 2.48s/it]
57%|█████████████████████████████████████████████████████████████████▍ | 391/681 [22:53<12:07, 2.51s/it]
{'loss': 0.6924, 'grad_norm': 53.62302017211914, 'learning_rate': 2.3076906145138405e-07, 'rewards/chosen': -0.7247334718704224, 'rewards/rejected': -1.8374460935592651, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.1127126216888428, 'logps/chosen': -198.61831665039062, 'logps/rejected': -493.40289306640625, 'logps/ref_chosen': -41.40957260131836, 'logps/ref_rejected': -93.46768951416016, 'logits/chosen': -2.1756703853607178, 'logits/rejected': -2.104757785797119, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.004595502279698849, 'epsilon_dpo/loss_margin_mean': 242.7264404296875, 'epsilon_dpo/beta_margin_mean': 1.1127126216888428, 'epsilon_dpo/beta_margin_std': 0.8292704820632935, 'epsilon_dpo/beta_margin_grad_mean': -0.273894727230072, 'epsilon_dpo/beta_margin_grad_std': 0.1488787978887558, 'kl/beta': 0.004632683005183935, 'kl/avg_steps': 0.8125, 'epoch': 0.57}
|
|||
|
|
57%|█████████████████████████████████████████████████████████████████▍ | 391/681 [22:53<12:07, 2.51s/it]
58%|█████████████████████████████████████████████████████████████████▌ | 392/681 [22:55<12:02, 2.50s/it]
{'loss': 0.5405, 'grad_norm': 46.09185791015625, 'learning_rate': 2.294897926507156e-07, 'rewards/chosen': -0.6257168054580688, 'rewards/rejected': -2.0817642211914062, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.456047534942627, 'logps/chosen': -166.53561401367188, 'logps/rejected': -566.6787109375, 'logps/ref_chosen': -29.435636520385742, 'logps/ref_rejected': -109.47046661376953, 'logits/chosen': -2.129703998565674, 'logits/rejected': -2.0612103939056396, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.004555592313408852, 'epsilon_dpo/loss_margin_mean': 320.1082763671875, 'epsilon_dpo/beta_margin_mean': 1.4560474157333374, 'epsilon_dpo/beta_margin_std': 0.9119820594787598, 'epsilon_dpo/beta_margin_grad_mean': -0.2217557728290558, 'epsilon_dpo/beta_margin_grad_std': 0.1372254192829132, 'kl/beta': 0.004595345817506313, 'kl/avg_steps': 0.875, 'epoch': 0.58}
|
|||
|
|
58%|█████████████████████████████████████████████████████████████████▌ | 392/681 [22:55<12:02, 2.50s/it]
58%|█████████████████████████████████████████████████████████████████▊ | 393/681 [22:58<12:06, 2.52s/it]
{'loss': 0.7035, 'grad_norm': 51.396175384521484, 'learning_rate': 2.2821106431308543e-07, 'rewards/chosen': -0.6019172668457031, 'rewards/rejected': -1.7369953393936157, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.1350780725479126, 'logps/chosen': -160.30789184570312, 'logps/rejected': -462.23565673828125, 'logps/ref_chosen': -27.59358024597168, 'logps/ref_rejected': -77.63695526123047, 'logits/chosen': -2.204120635986328, 'logits/rejected': -2.0319063663482666, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.004520347807556391, 'epsilon_dpo/loss_margin_mean': 251.88442993164062, 'epsilon_dpo/beta_margin_mean': 1.1350780725479126, 'epsilon_dpo/beta_margin_std': 0.9084685444831848, 'epsilon_dpo/beta_margin_grad_mean': -0.2750376760959625, 'epsilon_dpo/beta_margin_grad_std': 0.16051527857780457, 'kl/beta': 0.004555485676974058, 'kl/avg_steps': 0.78125, 'epoch': 0.58}
|
|||
|
|
58%|█████████████████████████████████████████████████████████████████▊ | 393/681 [22:58<12:06, 2.52s/it]
58%|█████████████████████████████████████████████████████████████████▉ | 394/681 [23:01<12:22, 2.59s/it]
{'loss': 0.5407, 'grad_norm': 36.82295227050781, 'learning_rate': 2.2693291013417452e-07, 'rewards/chosen': -0.61604905128479, 'rewards/rejected': -2.048534393310547, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.4324853420257568, 'logps/chosen': -171.11074829101562, 'logps/rejected': -554.876953125, 'logps/ref_chosen': -33.609615325927734, 'logps/ref_rejected': -97.24693298339844, 'logits/chosen': -2.1519179344177246, 'logits/rejected': -2.0851173400878906, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.004478242713958025, 'epsilon_dpo/loss_margin_mean': 320.1289367675781, 'epsilon_dpo/beta_margin_mean': 1.4324853420257568, 'epsilon_dpo/beta_margin_std': 0.8796045184135437, 'epsilon_dpo/beta_margin_grad_mean': -0.22447995841503143, 'epsilon_dpo/beta_margin_grad_std': 0.1293361932039261, 'kl/beta': 0.0045201717875897884, 'kl/avg_steps': 0.9375, 'epoch': 0.58}
|
|||
|
|
58%|█████████████████████████████████████████████████████████████████▉ | 394/681 [23:01<12:22, 2.59s/it]
58%|██████████████████████████████████████████████████████████████████ | 395/681 [23:03<12:21, 2.59s/it]
{'loss': 0.6914, 'grad_norm': 55.038551330566406, 'learning_rate': 2.2565536379453404e-07, 'rewards/chosen': -0.6987878084182739, 'rewards/rejected': -1.9468872547149658, 'rewards/accuracies': 0.875, 'rewards/margins': 1.248099446296692, 'logps/chosen': -198.61692810058594, 'logps/rejected': -528.0548095703125, 'logps/ref_chosen': -42.13022994995117, 'logps/ref_rejected': -89.85330963134766, 'logits/chosen': -2.1458773612976074, 'logits/rejected': -2.072359085083008, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.004446446429938078, 'epsilon_dpo/loss_margin_mean': 281.7148132324219, 'epsilon_dpo/beta_margin_mean': 1.248099446296692, 'epsilon_dpo/beta_margin_std': 1.0518742799758911, 'epsilon_dpo/beta_margin_grad_mean': -0.2658129632472992, 'epsilon_dpo/beta_margin_grad_std': 0.17667248845100403, 'kl/beta': 0.004478188697248697, 'kl/avg_steps': 0.71875, 'epoch': 0.58}
|
|||
|
|
58%|██████████████████████████████████████████████████████████████████ | 395/681 [23:03<12:21, 2.59s/it]
58%|██████████████████████████████████████████████████████████████████▎ | 396/681 [23:06<12:13, 2.57s/it]
{'loss': 0.6594, 'grad_norm': 54.677188873291016, 'learning_rate': 2.2437845895869825e-07, 'rewards/chosen': -0.706026017665863, 'rewards/rejected': -2.0237832069396973, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.3177571296691895, 'logps/chosen': -198.652099609375, 'logps/rejected': -553.473876953125, 'logps/ref_chosen': -39.016197204589844, 'logps/ref_rejected': -94.28376007080078, 'logits/chosen': -2.223626136779785, 'logits/rejected': -2.068613290786743, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.004410546738654375, 'epsilon_dpo/loss_margin_mean': 299.5541687011719, 'epsilon_dpo/beta_margin_mean': 1.317757248878479, 'epsilon_dpo/beta_margin_std': 1.0815587043762207, 'epsilon_dpo/beta_margin_grad_mean': -0.25534769892692566, 'epsilon_dpo/beta_margin_grad_std': 0.17317545413970947, 'kl/beta': 0.004446231294423342, 'kl/avg_steps': 0.8125, 'epoch': 0.58}
|
|||
|
|
58%|██████████████████████████████████████████████████████████████████▎ | 396/681 [23:06<12:13, 2.57s/it]
58%|██████████████████████████████████████████████████████████████████▍ | 397/681 [23:08<11:53, 2.51s/it]
{'loss': 0.5673, 'grad_norm': 48.635677337646484, 'learning_rate': 2.2310222927429716e-07, 'rewards/chosen': -0.6591649055480957, 'rewards/rejected': -2.1491518020629883, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.4899868965148926, 'logps/chosen': -192.5784149169922, 'logps/rejected': -603.2269287109375, 'logps/ref_chosen': -42.487823486328125, 'logps/ref_rejected': -111.77519226074219, 'logits/chosen': -2.2994096279144287, 'logits/rejected': -2.131638526916504, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.004374999552965164, 'epsilon_dpo/loss_margin_mean': 341.36114501953125, 'epsilon_dpo/beta_margin_mean': 1.489986777305603, 'epsilon_dpo/beta_margin_std': 1.04046630859375, 'epsilon_dpo/beta_margin_grad_mean': -0.2254161238670349, 'epsilon_dpo/beta_margin_grad_std': 0.1620989739894867, 'kl/beta': 0.004410396795719862, 'kl/avg_steps': 0.8125, 'epoch': 0.58}
|
|||
|
|
58%|██████████████████████████████████████████████████████████████████▍ | 397/681 [23:08<11:53, 2.51s/it]
58%|██████████████████████████████████████████████████████████████████▋ | 398/681 [23:10<11:43, 2.49s/it]
{'loss': 0.6398, 'grad_norm': 49.05772399902344, 'learning_rate': 2.2182670837116972e-07, 'rewards/chosen': -0.6942225694656372, 'rewards/rejected': -2.098356246948242, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.4041335582733154, 'logps/chosen': -206.82467651367188, 'logps/rejected': -600.200439453125, 'logps/ref_chosen': -47.66093444824219, 'logps/ref_rejected': -116.59112548828125, 'logits/chosen': -2.292893409729004, 'logits/rejected': -2.2275965213775635, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.004342474043369293, 'epsilon_dpo/loss_margin_mean': 324.445556640625, 'epsilon_dpo/beta_margin_mean': 1.4041334390640259, 'epsilon_dpo/beta_margin_std': 1.1149554252624512, 'epsilon_dpo/beta_margin_grad_mean': -0.2421146184206009, 'epsilon_dpo/beta_margin_grad_std': 0.1806570291519165, 'kl/beta': 0.004374851007014513, 'kl/avg_steps': 0.75, 'epoch': 0.58}
|
|||
|
|
58%|██████████████████████████████████████████████████████████████████▋ | 398/681 [23:10<11:43, 2.49s/it]
59%|██████████████████████████████████████████████████████████████████▊ | 399/681 [23:13<11:38, 2.48s/it]
{'loss': 0.6303, 'grad_norm': 54.13144302368164, 'learning_rate': 2.2055192986047804e-07, 'rewards/chosen': -0.5922940969467163, 'rewards/rejected': -1.9179613590240479, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.325667142868042, 'logps/chosen': -172.4034423828125, 'logps/rejected': -528.648193359375, 'logps/ref_chosen': -35.37146759033203, 'logps/ref_rejected': -83.08477020263672, 'logits/chosen': -2.2859010696411133, 'logits/rejected': -2.0517654418945312, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.004307433497160673, 'epsilon_dpo/loss_margin_mean': 308.531494140625, 'epsilon_dpo/beta_margin_mean': 1.3256672620773315, 'epsilon_dpo/beta_margin_std': 0.9880812168121338, 'epsilon_dpo/beta_margin_grad_mean': -0.24833783507347107, 'epsilon_dpo/beta_margin_grad_std': 0.16362829506397247, 'kl/beta': 0.004342284053564072, 'kl/avg_steps': 0.8125, 'epoch': 0.59}
|
|||
|
|
59%|██████████████████████████████████████████████████████████████████▊ | 399/681 [23:13<11:38, 2.48s/it]
59%|██████████████████████████████████████████████████████████████████▉ | 400/681 [23:15<11:42, 2.50s/it]
{'loss': 0.5665, 'grad_norm': 42.21733093261719, 'learning_rate': 2.192779273338215e-07, 'rewards/chosen': -0.6924352645874023, 'rewards/rejected': -2.0552854537963867, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.3628504276275635, 'logps/chosen': -214.174560546875, 'logps/rejected': -592.43896484375, 'logps/ref_chosen': -52.37220001220703, 'logps/ref_rejected': -110.96676635742188, 'logits/chosen': -2.346558094024658, 'logits/rejected': -2.2209692001342773, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0042700255289673805, 'epsilon_dpo/loss_margin_mean': 319.66986083984375, 'epsilon_dpo/beta_margin_mean': 1.362850308418274, 'epsilon_dpo/beta_margin_std': 0.8101769089698792, 'epsilon_dpo/beta_margin_grad_mean': -0.23046433925628662, 'epsilon_dpo/beta_margin_grad_std': 0.13983549177646637, 'kl/beta': 0.0043072872795164585, 'kl/avg_steps': 0.875, 'epoch': 0.59}
|
|||
|
|
59%|██████████████████████████████████████████████████████████████████▉ | 400/681 [23:16<11:42, 2.50s/it][INFO|trainer.py:4307] 2026-04-18 09:54:37,937 >>
|
|||
|
|
***** Running Evaluation *****
|
|||
|
|
[INFO|trainer.py:4309] 2026-04-18 09:54:37,937 >> Num examples = 2339
|
|||
|
|
[INFO|trainer.py:4312] 2026-04-18 09:54:37,937 >> Batch size = 8
|
|||
|
|
|
|||
|
|
0%| | 0/73 [00:00<?, ?it/s][A
|
|||
|
|
3%|███▏ | 2/73 [00:00<00:18, 3.86it/s][A
|
|||
|
|
4%|████▊ | 3/73 [00:01<00:26, 2.61it/s][A
|
|||
|
|
5%|██████▍ | 4/73 [00:01<00:30, 2.25it/s][A
|
|||
|
|
7%|████████ | 5/73 [00:02<00:32, 2.09it/s][A
|
|||
|
|
8%|█████████▌ | 6/73 [00:02<00:33, 2.02it/s][A
|
|||
|
|
10%|███████████▏ | 7/73 [00:03<00:32, 2.00it/s][A
|
|||
|
|
11%|████████████▊ | 8/73 [00:03<00:31, 2.06it/s][A
|
|||
|
|
12%|██████████████▍ | 9/73 [00:04<00:32, 2.00it/s][A
|
|||
|
|
14%|███████████████▉ | 10/73 [00:04<00:32, 1.95it/s][A
|
|||
|
|
15%|█████████████████▍ | 11/73 [00:05<00:31, 1.95it/s][A
|
|||
|
|
16%|███████████████████ | 12/73 [00:05<00:31, 1.96it/s][A
|
|||
|
|
18%|████████████████████▋ | 13/73 [00:06<00:31, 1.91it/s][A
|
|||
|
|
19%|██████████████████████▏ | 14/73 [00:06<00:30, 1.91it/s][A
|
|||
|
|
21%|███████████████████████▊ | 15/73 [00:07<00:31, 1.87it/s][A
|
|||
|
|
22%|█████████████████████████▍ | 16/73 [00:07<00:30, 1.85it/s][A
|
|||
|
|
23%|███████████████████████████ | 17/73 [00:08<00:29, 1.87it/s][A
|
|||
|
|
25%|████████████████████████████▌ | 18/73 [00:09<00:29, 1.85it/s][A
|
|||
|
|
26%|██████████████████████████████▏ | 19/73 [00:09<00:29, 1.84it/s][A
|
|||
|
|
27%|███████████████████████████████▊ | 20/73 [00:10<00:29, 1.83it/s][A
|
|||
|
|
29%|█████████████████████████████████▎ | 21/73 [00:10<00:28, 1.82it/s][A
|
|||
|
|
30%|██████████████████████████████████▉ | 22/73 [00:11<00:28, 1.82it/s][A
|
|||
|
|
32%|████████████████████████████████████▌ | 23/73 [00:11<00:27, 1.84it/s][A
|
|||
|
|
33%|██████████████████████████████████████▏ | 24/73 [00:12<00:26, 1.88it/s][A
|
|||
|
|
34%|███████████████████████████████████████▋ | 25/73 [00:12<00:25, 1.87it/s][A
|
|||
|
|
36%|█████████████████████████████████████████▎ | 26/73 [00:13<00:24, 1.91it/s][A
|
|||
|
|
37%|██████████████████████████████████████████▉ | 27/73 [00:13<00:22, 2.05it/s][A
|
|||
|
|
38%|████████████████████████████████████████████▍ | 28/73 [00:14<00:21, 2.05it/s][A
|
|||
|
|
40%|██████████████████████████████████████████████ | 29/73 [00:14<00:21, 2.01it/s][A
|
|||
|
|
41%|███████████████████████████████████████████████▋ | 30/73 [00:15<00:21, 1.97it/s][A
|
|||
|
|
42%|█████████████████████████████████████████████████▎ | 31/73 [00:15<00:21, 1.93it/s][A
|
|||
|
|
44%|██████████████████████████████████████████████████▊ | 32/73 [00:16<00:21, 1.95it/s][A
|
|||
|
|
45%|████████████████████████████████████████████████████▍ | 33/73 [00:16<00:20, 1.94it/s][A
|
|||
|
|
47%|██████████████████████████████████████████████████████ | 34/73 [00:17<00:19, 1.97it/s][A
|
|||
|
|
48%|███████████████████████████████████████████████████████▌ | 35/73 [00:17<00:19, 1.90it/s][A
|
|||
|
|
49%|█████████████████████████████████████████████████████████▏ | 36/73 [00:18<00:19, 1.91it/s][A
|
|||
|
|
51%|██████████████████████████████████████████████████████████▊ | 37/73 [00:18<00:19, 1.87it/s][A
|
|||
|
|
52%|████████████████████████████████████████████████████████████▍ | 38/73 [00:19<00:18, 1.93it/s][A
|
|||
|
|
53%|█████████████████████████████████████████████████████████████▉ | 39/73 [00:19<00:17, 1.92it/s][A
|
|||
|
|
55%|███████████████████████████████████████████████████████████████▌ | 40/73 [00:20<00:17, 1.92it/s][A
|
|||
|
|
56%|█████████████████████████████████████████████████████████████████▏ | 41/73 [00:20<00:16, 1.94it/s][A
|
|||
|
|
58%|██████████████████████████████████████████████████████████████████▋ | 42/73 [00:21<00:16, 1.90it/s][A
|
|||
|
|
59%|████████████████████████████████████████████████████████████████████▎ | 43/73 [00:22<00:15, 1.88it/s][A
|
|||
|
|
60%|█████████████████████████████████████████████████████████████████████▉ | 44/73 [00:22<00:15, 1.86it/s][A
|
|||
|
|
62%|███████████████████████████████████████████████████████████████████████▌ | 45/73 [00:23<00:15, 1.83it/s][A
|
|||
|
|
63%|█████████████████████████████████████████████████████████████████████████ | 46/73 [00:23<00:14, 1.87it/s][A
|
|||
|
|
64%|██████████████████████████████████████████████████████████████████████████▋ | 47/73 [00:24<00:13, 1.89it/s][A
|
|||
|
|
66%|████████████████████████████████████████████████████████████████████████████▎ | 48/73 [00:24<00:12, 1.93it/s][A
|
|||
|
|
67%|█████████████████████████████████████████████████████████████████████████████▊ | 49/73 [00:25<00:12, 1.88it/s][A
|
|||
|
|
68%|███████████████████████████████████████████████████████████████████████████████▍ | 50/73 [00:25<00:12, 1.86it/s][A
|
|||
|
|
70%|█████████████████████████████████████████████████████████████████████████████████ | 51/73 [00:26<00:11, 1.84it/s][A
|
|||
|
|
71%|██████████████████████████████████████████████████████████████████████████████████▋ | 52/73 [00:26<00:11, 1.83it/s][A
|
|||
|
|
73%|████████████████████████████████████████████████████████████████████████████████████▏ | 53/73 [00:27<00:10, 1.94it/s][A
|
|||
|
|
74%|█████████████████████████████████████████████████████████████████████████████████████▊ | 54/73 [00:27<00:09, 1.97it/s][A
|
|||
|
|
75%|███████████████████████████████████████████████████████████████████████████████████████▍ | 55/73 [00:28<00:08, 2.01it/s][A
|
|||
|
|
77%|████████████████████████████████████████████████████████████████████████████████████████▉ | 56/73 [00:28<00:08, 1.95it/s][A
|
|||
|
|
78%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 57/73 [00:29<00:08, 1.91it/s][A
|
|||
|
|
79%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 58/73 [00:29<00:07, 1.94it/s][A
|
|||
|
|
81%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 59/73 [00:30<00:07, 1.89it/s][A
|
|||
|
|
82%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 60/73 [00:31<00:06, 1.89it/s][A
|
|||
|
|
84%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 61/73 [00:31<00:06, 1.87it/s][A
|
|||
|
|
85%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 62/73 [00:32<00:05, 1.85it/s][A
|
|||
|
|
86%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 63/73 [00:32<00:05, 1.91it/s][A
|
|||
|
|
88%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 64/73 [00:33<00:04, 1.93it/s][A
|
|||
|
|
89%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 65/73 [00:33<00:04, 1.88it/s][A
|
|||
|
|
90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 66/73 [00:34<00:03, 1.87it/s][A
|
|||
|
|
92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 67/73 [00:34<00:03, 1.87it/s][A
|
|||
|
|
93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 68/73 [00:35<00:02, 1.87it/s][A
|
|||
|
|
95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 69/73 [00:35<00:02, 1.99it/s][A
|
|||
|
|
96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 70/73 [00:36<00:01, 1.93it/s][A
|
|||
|
|
97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 71/73 [00:36<00:01, 1.91it/s][A
|
|||
|
|
99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 72/73 [00:37<00:00, 1.89it/s][A
|
|||
|
|
100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 73/73 [00:37<00:00, 2.00it/s][A
|
|||
|
|
[A{'eval_loss': 0.4863179922103882, 'eval_runtime': 38.3555, 'eval_samples_per_second': 60.982, 'eval_steps_per_second': 1.929, 'eval_epsilon_dpo/beta': 0.004249695222824812, 'eval_epsilon_dpo/loss_margin_mean': 180.02191162109375, 'eval_epsilon_dpo/beta_margin_mean': 0.7586736679077148, 'eval_epsilon_dpo/beta_margin_std': 0.9838672876358032, 'eval_epsilon_dpo/beta_margin_grad_mean': -0.35010644793510437, 'eval_epsilon_dpo/beta_margin_grad_std': 0.18771161139011383, 'eval_rewards/chosen': -1.077276587486267, 'eval_rewards/rejected': -1.835950255393982, 'eval_rewards/accuracies': 0.7564212083816528, 'eval_rewards/margins': 0.7586737275123596, 'eval_logps/chosen': -320.8791198730469, 'eval_logps/rejected': -524.6903076171875, 'eval_logps/ref_chosen': -68.29110717773438, 'eval_logps/ref_rejected': -92.08038330078125, 'eval_logits/chosen': -2.246497631072998, 'eval_logits/rejected': -2.1281936168670654, 'eval_kl/p_epsilon_steps': 0.741866409778595, 'eval_kl/n_epsilon_steps': 0.25813356041908264, 'epoch': 0.59}
|
|||
|
|
59%|██████████████████████████████████████████████████████████████████▉ | 400/681 [23:54<11:42, 2.50s/it]
|
|||
|
|
100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 73/73 [00:37<00:00, 2.00it/s][A
|
|||
|
|
[A[INFO|trainer.py:3984] 2026-04-18 09:55:32,397 >> Saving model checkpoint to /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-400
|
|||
|
|
[INFO|configuration_utils.py:419] 2026-04-18 09:55:32,405 >> Configuration saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-400/config.json
|
|||
|
|
[INFO|configuration_utils.py:911] 2026-04-18 09:55:32,414 >> Configuration saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-400/generation_config.json
|
|||
|
|
[INFO|modeling_utils.py:3580] 2026-04-18 09:56:24,111 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 6 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-400/model.safetensors.index.json.
|
|||
|
|
[INFO|tokenization_utils_base.py:2510] 2026-04-18 09:56:24,151 >> tokenizer config file saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-400/tokenizer_config.json
|
|||
|
|
[INFO|tokenization_utils_base.py:2519] 2026-04-18 09:56:24,160 >> Special tokens file saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-400/special_tokens_map.json
|
|||
|
|
59%|█████████████████████████████████████████████████████████████████▉ | 401/681 [28:18<7:12:09, 92.60s/it]
{'loss': 0.7457, 'grad_norm': 55.85029983520508, 'learning_rate': 2.1800473436235136e-07, 'rewards/chosen': -0.7069000005722046, 'rewards/rejected': -1.8603973388671875, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.153497338294983, 'logps/chosen': -206.0033721923828, 'logps/rejected': -530.2316284179688, 'logps/ref_chosen': -39.69450378417969, 'logps/ref_rejected': -90.86283874511719, 'logits/chosen': -2.2151618003845215, 'logits/rejected': -2.1000635623931885, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0042383247055113316, 'epsilon_dpo/loss_margin_mean': 273.0599365234375, 'epsilon_dpo/beta_margin_mean': 1.153497338294983, 'epsilon_dpo/beta_margin_std': 1.093888521194458, 'epsilon_dpo/beta_margin_grad_mean': -0.2859787940979004, 'epsilon_dpo/beta_margin_grad_std': 0.17702849209308624, 'kl/beta': 0.004269925411790609, 'kl/avg_steps': 0.75, 'epoch': 0.59}
|
|||
|
|
59%|█████████████████████████████████████████████████████████████████▉ | 401/681 [28:18<7:12:09, 92.60s/it]
59%|██████████████████████████████████████████████████████████████████ | 402/681 [28:21<5:04:44, 65.53s/it]
{'loss': 0.6286, 'grad_norm': 44.19596862792969, 'learning_rate': 2.1673238449588665e-07, 'rewards/chosen': -0.6281874179840088, 'rewards/rejected': -1.8827701807022095, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.2545827627182007, 'logps/chosen': -187.8063201904297, 'logps/rejected': -534.6738891601562, 'logps/ref_chosen': -38.76295852661133, 'logps/ref_rejected': -86.50106811523438, 'logits/chosen': -2.216102123260498, 'logits/rejected': -2.0876975059509277, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.004204125143587589, 'epsilon_dpo/loss_margin_mean': 299.1294860839844, 'epsilon_dpo/beta_margin_mean': 1.2545827627182007, 'epsilon_dpo/beta_margin_std': 0.8603823184967041, 'epsilon_dpo/beta_margin_grad_mean': -0.251901239156723, 'epsilon_dpo/beta_margin_grad_std': 0.14963571727275848, 'kl/beta': 0.004238139372318983, 'kl/avg_steps': 0.8125, 'epoch': 0.59}
|
|||
|
|
59%|██████████████████████████████████████████████████████████████████ | 402/681 [28:21<5:04:44, 65.53s/it]
59%|██████████████████████████████████████████████████████████████████▎ | 403/681 [28:23<3:36:01, 46.62s/it]
{'loss': 0.6468, 'grad_norm': 45.329769134521484, 'learning_rate': 2.154609112620295e-07, 'rewards/chosen': -0.5560154914855957, 'rewards/rejected': -1.8019505739212036, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.2459352016448975, 'logps/chosen': -162.46539306640625, 'logps/rejected': -515.2846069335938, 'logps/ref_chosen': -29.60453224182129, 'logps/ref_rejected': -82.97395324707031, 'logits/chosen': -2.254790782928467, 'logits/rejected': -2.070467472076416, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.004170241765677929, 'epsilon_dpo/loss_margin_mean': 299.4497985839844, 'epsilon_dpo/beta_margin_mean': 1.245935082435608, 'epsilon_dpo/beta_margin_std': 0.9069526791572571, 'epsilon_dpo/beta_margin_grad_mean': -0.2570376694202423, 'epsilon_dpo/beta_margin_grad_std': 0.15749378502368927, 'kl/beta': 0.004203982185572386, 'kl/avg_steps': 0.8125, 'epoch': 0.59}
|
|||
|
|
59%|██████████████████████████████████████████████████████████████████▎ | 403/681 [28:23<3:36:01, 46.62s/it]
59%|██████████████████████████████████████████████████████████████████▍ | 404/681 [28:26<2:34:06, 33.38s/it]
{'loss': 0.7084, 'grad_norm': 52.8735466003418, 'learning_rate': 2.1419034816528218e-07, 'rewards/chosen': -0.6246769428253174, 'rewards/rejected': -1.7911345958709717, 'rewards/accuracies': 0.859375, 'rewards/margins': 1.1664576530456543, 'logps/chosen': -182.46783447265625, 'logps/rejected': -517.0897216796875, 'logps/ref_chosen': -32.369415283203125, 'logps/ref_rejected': -84.27439880371094, 'logits/chosen': -2.253624439239502, 'logits/rejected': -2.092195987701416, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.004143148194998503, 'epsilon_dpo/loss_margin_mean': 282.7168884277344, 'epsilon_dpo/beta_margin_mean': 1.1664576530456543, 'epsilon_dpo/beta_margin_std': 0.9855588674545288, 'epsilon_dpo/beta_margin_grad_mean': -0.2758185565471649, 'epsilon_dpo/beta_margin_grad_std': 0.16760212182998657, 'kl/beta': 0.004170100204646587, 'kl/avg_steps': 0.65625, 'epoch': 0.59}
|
|||
|
|
59%|██████████████████████████████████████████████████████████████████▍ | 404/681 [28:26<2:34:06, 33.38s/it]
59%|██████████████████████████████████████████████████████████████████▌ | 405/681 [28:28<1:51:01, 24.13s/it]
{'loss': 0.6822, 'grad_norm': 45.45560836791992, 'learning_rate': 2.129207286861638e-07, 'rewards/chosen': -0.6669937372207642, 'rewards/rejected': -1.8423340320587158, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.1753404140472412, 'logps/chosen': -207.39706420898438, 'logps/rejected': -542.759033203125, 'logps/ref_chosen': -45.16187286376953, 'logps/ref_rejected': -93.87014770507812, 'logits/chosen': -2.2662670612335205, 'logits/rejected': -2.1083085536956787, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.004105777945369482, 'epsilon_dpo/loss_margin_mean': 286.6537170410156, 'epsilon_dpo/beta_margin_mean': 1.1753402948379517, 'epsilon_dpo/beta_margin_std': 0.9294463396072388, 'epsilon_dpo/beta_margin_grad_mean': -0.27076855301856995, 'epsilon_dpo/beta_margin_grad_std': 0.15409667789936066, 'kl/beta': 0.004142912104725838, 'kl/avg_steps': 0.90625, 'epoch': 0.59}
|
|||
|
|
59%|██████████████████████████████████████████████████████████████████▌ | 405/681 [28:28<1:51:01, 24.13s/it]
60%|██████████████████████████████████████████████████████████████████▊ | 406/681 [28:31<1:21:08, 17.70s/it]
{'loss': 0.612, 'grad_norm': 44.63689422607422, 'learning_rate': 2.1165208628032861e-07, 'rewards/chosen': -0.5877770185470581, 'rewards/rejected': -1.8649117946624756, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.2771347761154175, 'logps/chosen': -175.9896240234375, 'logps/rejected': -557.4638671875, 'logps/ref_chosen': -32.108238220214844, 'logps/ref_rejected': -99.3056640625, 'logits/chosen': -2.2001147270202637, 'logits/rejected': -2.1016151905059814, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.004071469884365797, 'epsilon_dpo/loss_margin_mean': 314.2767639160156, 'epsilon_dpo/beta_margin_mean': 1.2771347761154175, 'epsilon_dpo/beta_margin_std': 0.8445379137992859, 'epsilon_dpo/beta_margin_grad_mean': -0.24695971608161926, 'epsilon_dpo/beta_margin_grad_std': 0.14529289305210114, 'kl/beta': 0.004105704370886087, 'kl/avg_steps': 0.84375, 'epoch': 0.6}
|
|||
|
|
60%|██████████████████████████████████████████████████████████████████▊ | 406/681 [28:31<1:21:08, 17.70s/it]
60%|██████████████████████████████████████████████████████████████████▉ | 407/681 [28:33<1:00:03, 13.15s/it]
{'loss': 0.7582, 'grad_norm': 52.973270416259766, 'learning_rate': 2.1038445437768375e-07, 'rewards/chosen': -0.5955917835235596, 'rewards/rejected': -1.6564050912857056, 'rewards/accuracies': 0.859375, 'rewards/margins': 1.060813307762146, 'logps/chosen': -181.23577880859375, 'logps/rejected': -493.41815185546875, 'logps/ref_chosen': -34.63081359863281, 'logps/ref_rejected': -83.40669250488281, 'logits/chosen': -2.2982513904571533, 'logits/rejected': -2.1020822525024414, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0040450384840369225, 'epsilon_dpo/loss_margin_mean': 263.4065246582031, 'epsilon_dpo/beta_margin_mean': 1.060813307762146, 'epsilon_dpo/beta_margin_std': 0.9306771159172058, 'epsilon_dpo/beta_margin_grad_mean': -0.2901814579963684, 'epsilon_dpo/beta_margin_grad_std': 0.17480535805225372, 'kl/beta': 0.004071352072060108, 'kl/avg_steps': 0.65625, 'epoch': 0.6}
|
|||
|
|
60%|██████████████████████████████████████████████████████████████████▉ | 407/681 [28:33<1:00:03, 13.15s/it]
60%|████████████████████████████████████████████████████████████████████▎ | 408/681 [28:36<45:21, 9.97s/it]
{'loss': 0.682, 'grad_norm': 44.9481201171875, 'learning_rate': 2.0911786638150872e-07, 'rewards/chosen': -0.6118342280387878, 'rewards/rejected': -1.7125052213668823, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.1006710529327393, 'logps/chosen': -198.23095703125, 'logps/rejected': -523.3738403320312, 'logps/ref_chosen': -46.1392822265625, 'logps/ref_rejected': -96.3233642578125, 'logits/chosen': -2.361016273498535, 'logits/rejected': -2.139800548553467, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.004012344870716333, 'epsilon_dpo/loss_margin_mean': 274.95880126953125, 'epsilon_dpo/beta_margin_mean': 1.1006709337234497, 'epsilon_dpo/beta_margin_std': 0.758716344833374, 'epsilon_dpo/beta_margin_grad_mean': -0.2729249894618988, 'epsilon_dpo/beta_margin_grad_std': 0.14225371181964874, 'kl/beta': 0.004044807981699705, 'kl/avg_steps': 0.8125, 'epoch': 0.6}
|
|||
|
|
60%|████████████████████████████████████████████████████████████████████▎ | 408/681 [28:36<45:21, 9.97s/it]
60%|████████████████████████████████████████████████████████████████████▍ | 409/681 [28:39<35:10, 7.76s/it]
{'loss': 0.7654, 'grad_norm': 45.330448150634766, 'learning_rate': 2.0785235566757517e-07, 'rewards/chosen': -0.6647806167602539, 'rewards/rejected': -1.6724402904510498, 'rewards/accuracies': 0.859375, 'rewards/margins': 1.007659673690796, 'logps/chosen': -214.66848754882812, 'logps/rejected': -508.6387939453125, 'logps/ref_chosen': -48.41924285888672, 'logps/ref_rejected': -88.46084594726562, 'logits/chosen': -2.2953333854675293, 'logits/rejected': -2.1765618324279785, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.003982515539973974, 'epsilon_dpo/loss_margin_mean': 253.92869567871094, 'epsilon_dpo/beta_margin_mean': 1.007659673690796, 'epsilon_dpo/beta_margin_std': 0.8669053316116333, 'epsilon_dpo/beta_margin_grad_mean': -0.294739693403244, 'epsilon_dpo/beta_margin_grad_std': 0.16220302879810333, 'kl/beta': 0.0040122088976204395, 'kl/avg_steps': 0.75, 'epoch': 0.6}
|
|||
|
|
60%|████████████████████████████████████████████████████████████████████▍ | 409/681 [28:39<35:10, 7.76s/it]
60%|████████████████████████████████████████████████████████████████████▋ | 410/681 [28:41<27:56, 6.19s/it]
{'loss': 0.6897, 'grad_norm': 39.14712905883789, 'learning_rate': 2.065879555832674e-07, 'rewards/chosen': -0.5501143336296082, 'rewards/rejected': -1.676256775856018, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.1261425018310547, 'logps/chosen': -171.04891967773438, 'logps/rejected': -515.6138916015625, 'logps/ref_chosen': -32.20702362060547, 'logps/ref_rejected': -90.97166442871094, 'logits/chosen': -2.258359909057617, 'logits/rejected': -2.2356643676757812, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.003950380254536867, 'epsilon_dpo/loss_margin_mean': 285.80035400390625, 'epsilon_dpo/beta_margin_mean': 1.1261425018310547, 'epsilon_dpo/beta_margin_std': 0.8273832201957703, 'epsilon_dpo/beta_margin_grad_mean': -0.27121517062187195, 'epsilon_dpo/beta_margin_grad_std': 0.15460430085659027, 'kl/beta': 0.003982341382652521, 'kl/avg_steps': 0.8125, 'epoch': 0.6}
|
|||
|
|
60%|████████████████████████████████████████████████████████████████████▋ | 410/681 [28:41<27:56, 6.19s/it]
60%|████████████████████████████████████████████████████████████████████▊ | 411/681 [28:44<22:43, 5.05s/it]
{'loss': 0.6557, 'grad_norm': 38.4173698425293, 'learning_rate': 2.0532469944670343e-07, 'rewards/chosen': -0.5305934548377991, 'rewards/rejected': -1.6526423692703247, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.1220488548278809, 'logps/chosen': -163.12564086914062, 'logps/rejected': -509.96392822265625, 'logps/ref_chosen': -27.866039276123047, 'logps/ref_rejected': -87.75438690185547, 'logits/chosen': -2.2671849727630615, 'logits/rejected': -2.18176007270813, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0039160726591944695, 'epsilon_dpo/loss_margin_mean': 286.949951171875, 'epsilon_dpo/beta_margin_mean': 1.1220488548278809, 'epsilon_dpo/beta_margin_std': 0.7217124104499817, 'epsilon_dpo/beta_margin_grad_mean': -0.2674121856689453, 'epsilon_dpo/beta_margin_grad_std': 0.12682108581066132, 'kl/beta': 0.003950245678424835, 'kl/avg_steps': 0.875, 'epoch': 0.6}
|
|||
|
|
60%|████████████████████████████████████████████████████████████████████▊ | 411/681 [28:44<22:43, 5.05s/it]
60%|████████████████████████████████████████████████████████████████████▉ | 412/681 [28:46<19:03, 4.25s/it]
{'loss': 0.6232, 'grad_norm': 39.3294563293457, 'learning_rate': 2.0406262054585738e-07, 'rewards/chosen': -0.4781460464000702, 'rewards/rejected': -1.7018874883651733, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.2237414121627808, 'logps/chosen': -154.23080444335938, 'logps/rejected': -545.803466796875, 'logps/ref_chosen': -31.307266235351562, 'logps/ref_rejected': -107.21038818359375, 'logits/chosen': -2.2864794731140137, 'logits/rejected': -2.262669563293457, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0038821042980998755, 'epsilon_dpo/loss_margin_mean': 315.66949462890625, 'epsilon_dpo/beta_margin_mean': 1.2237414121627808, 'epsilon_dpo/beta_margin_std': 0.7976933121681213, 'epsilon_dpo/beta_margin_grad_mean': -0.254239022731781, 'epsilon_dpo/beta_margin_grad_std': 0.13515068590641022, 'kl/beta': 0.00391598092392087, 'kl/avg_steps': 0.875, 'epoch': 0.6}
|
|||
|
|
60%|████████████████████████████████████████████████████████████████████▉ | 412/681 [28:46<19:03, 4.25s/it]
61%|█████████████████████████████████████████████████████████████████████▏ | 413/681 [28:48<16:34, 3.71s/it]
{'loss': 0.6576, 'grad_norm': 38.31694412231445, 'learning_rate': 2.0280175213768205e-07, 'rewards/chosen': -0.5653949975967407, 'rewards/rejected': -1.7393302917480469, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.1739351749420166, 'logps/chosen': -189.5083465576172, 'logps/rejected': -558.2633666992188, 'logps/ref_chosen': -43.01777648925781, 'logps/ref_rejected': -106.10704040527344, 'logits/chosen': -2.353483200073242, 'logits/rejected': -2.2761778831481934, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.003849643748253584, 'epsilon_dpo/loss_margin_mean': 305.6657409667969, 'epsilon_dpo/beta_margin_mean': 1.1739351749420166, 'epsilon_dpo/beta_margin_std': 0.7997394800186157, 'epsilon_dpo/beta_margin_grad_mean': -0.2595198452472687, 'epsilon_dpo/beta_margin_grad_std': 0.14788804948329926, 'kl/beta': 0.003882013261318207, 'kl/avg_steps': 0.84375, 'epoch': 0.61}
|
|||
|
|
61%|█████████████████████████████████████████████████████████████████████▏ | 413/681 [28:48<16:34, 3.71s/it]
61%|█████████████████████████████████████████████████████████████████████▎ | 414/681 [28:51<15:00, 3.37s/it]
{'loss': 0.6491, 'grad_norm': 46.121341705322266, 'learning_rate': 2.0154212744723247e-07, 'rewards/chosen': -0.4871448874473572, 'rewards/rejected': -1.6691176891326904, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.181972861289978, 'logps/chosen': -161.10008239746094, 'logps/rejected': -531.1929931640625, 'logps/ref_chosen': -33.92742919921875, 'logps/ref_rejected': -93.77487182617188, 'logits/chosen': -2.2362513542175293, 'logits/rejected': -2.2514965534210205, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0038186372257769108, 'epsilon_dpo/loss_margin_mean': 310.2454833984375, 'epsilon_dpo/beta_margin_mean': 1.181972861289978, 'epsilon_dpo/beta_margin_std': 0.8064230680465698, 'epsilon_dpo/beta_margin_grad_mean': -0.2614062428474426, 'epsilon_dpo/beta_margin_grad_std': 0.14156121015548706, 'kl/beta': 0.003849532688036561, 'kl/avg_steps': 0.8125, 'epoch': 0.61}
|
|||
|
|
61%|█████████████████████████████████████████████████████████████████████▎ | 414/681 [28:51<15:00, 3.37s/it]
61%|█████████████████████████████████████████████████████████████████████▍ | 415/681 [28:54<13:58, 3.15s/it]
{'loss': 0.7054, 'grad_norm': 42.91494369506836, 'learning_rate': 2.002837796667909e-07, 'rewards/chosen': -0.5699288845062256, 'rewards/rejected': -1.6337485313415527, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.0638197660446167, 'logps/chosen': -195.28121948242188, 'logps/rejected': -539.98828125, 'logps/ref_chosen': -45.32706832885742, 'logps/ref_rejected': -108.54624938964844, 'logits/chosen': -2.451190948486328, 'logits/rejected': -2.4177732467651367, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0037890539970248938, 'epsilon_dpo/loss_margin_mean': 281.4879150390625, 'epsilon_dpo/beta_margin_mean': 1.0638197660446167, 'epsilon_dpo/beta_margin_std': 0.7809419631958008, 'epsilon_dpo/beta_margin_grad_mean': -0.281541645526886, 'epsilon_dpo/beta_margin_grad_std': 0.14272256195545197, 'kl/beta': 0.003818507306277752, 'kl/avg_steps': 0.78125, 'epoch': 0.61}
|
|||
|
|
61%|█████████████████████████████████████████████████████████████████████▍ | 415/681 [28:54<13:58, 3.15s/it]
61%|█████████████████████████████████████████████████████████████████████▋ | 416/681 [28:56<13:07, 2.97s/it]
{'loss': 0.6203, 'grad_norm': 41.69633483886719, 'learning_rate': 1.990267419549914e-07, 'rewards/chosen': -0.4858553111553192, 'rewards/rejected': -1.71781587600708, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.2319605350494385, 'logps/chosen': -167.67477416992188, 'logps/rejected': -555.6788330078125, 'logps/ref_chosen': -38.674949645996094, 'logps/ref_rejected': -98.17755889892578, 'logits/chosen': -2.335287094116211, 'logits/rejected': -2.2950291633605957, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.00375731335952878, 'epsilon_dpo/loss_margin_mean': 328.50146484375, 'epsilon_dpo/beta_margin_mean': 1.231960654258728, 'epsilon_dpo/beta_margin_std': 0.7866246700286865, 'epsilon_dpo/beta_margin_grad_mean': -0.2511499524116516, 'epsilon_dpo/beta_margin_grad_std': 0.1389520764350891, 'kl/beta': 0.003788906615227461, 'kl/avg_steps': 0.84375, 'epoch': 0.61}
|
|||
|
|
61%|█████████████████████████████████████████████████████████████████████▋ | 416/681 [28:56<13:07, 2.97s/it]
61%|█████████████████████████████████████████████████████████████████████▊ | 417/681 [28:59<12:28, 2.84s/it]
{'loss': 0.6858, 'grad_norm': 43.83847427368164, 'learning_rate': 1.9777104743594686e-07, 'rewards/chosen': -0.4534587562084198, 'rewards/rejected': -1.5868133306503296, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.1333545446395874, 'logps/chosen': -147.36915588378906, 'logps/rejected': -500.62115478515625, 'logps/ref_chosen': -26.17800521850586, 'logps/ref_rejected': -74.53215026855469, 'logits/chosen': -2.3649911880493164, 'logits/rejected': -2.098527431488037, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0037270504981279373, 'epsilon_dpo/loss_margin_mean': 304.8978271484375, 'epsilon_dpo/beta_margin_mean': 1.1333545446395874, 'epsilon_dpo/beta_margin_std': 0.8189650774002075, 'epsilon_dpo/beta_margin_grad_mean': -0.26951104402542114, 'epsilon_dpo/beta_margin_grad_std': 0.15652604401111603, 'kl/beta': 0.003757205093279481, 'kl/avg_steps': 0.8125, 'epoch': 0.61}
|
|||
|
|
61%|█████████████████████████████████████████████████████████████████████▊ | 417/681 [28:59<12:28, 2.84s/it]
61%|█████████████████████████████████████████████████████████████████████▉ | 418/681 [29:01<12:23, 2.83s/it]
{'loss': 0.6258, 'grad_norm': 42.56073760986328, 'learning_rate': 1.965167291983757e-07, 'rewards/chosen': -0.47949230670928955, 'rewards/rejected': -1.6942154169082642, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.2147231101989746, 'logps/chosen': -180.94284057617188, 'logps/rejected': -571.2081298828125, 'logps/ref_chosen': -51.68669128417969, 'logps/ref_rejected': -112.65290832519531, 'logits/chosen': -2.3986239433288574, 'logits/rejected': -2.3206000328063965, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0036970123182982206, 'epsilon_dpo/loss_margin_mean': 329.29913330078125, 'epsilon_dpo/beta_margin_mean': 1.2147231101989746, 'epsilon_dpo/beta_margin_std': 0.7788877487182617, 'epsilon_dpo/beta_margin_grad_mean': -0.25392869114875793, 'epsilon_dpo/beta_margin_grad_std': 0.1371525526046753, 'kl/beta': 0.0037269238382577896, 'kl/avg_steps': 0.8125, 'epoch': 0.61}
|
|||
|
|
61%|█████████████████████████████████████████████████████████████████████▉ | 418/681 [29:01<12:23, 2.83s/it]
62%|██████████████████████████████████████████████████████████████████████▏ | 419/681 [29:04<11:51, 2.71s/it]
{'loss': 0.6311, 'grad_norm': 44.2584228515625, 'learning_rate': 1.9526382029472988e-07, 'rewards/chosen': -0.4797900915145874, 'rewards/rejected': -1.664259910583496, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.1844696998596191, 'logps/chosen': -165.10519409179688, 'logps/rejected': -552.49658203125, 'logps/ref_chosen': -34.45082473754883, 'logps/ref_rejected': -98.03851318359375, 'logits/chosen': -2.3918709754943848, 'logits/rejected': -2.2970142364501953, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.003663749899715185, 'epsilon_dpo/loss_margin_mean': 323.8036804199219, 'epsilon_dpo/beta_margin_mean': 1.1844698190689087, 'epsilon_dpo/beta_margin_std': 0.7216576337814331, 'epsilon_dpo/beta_margin_grad_mean': -0.25385352969169617, 'epsilon_dpo/beta_margin_grad_std': 0.13596518337726593, 'kl/beta': 0.0036968865897506475, 'kl/avg_steps': 0.90625, 'epoch': 0.62}
|
|||
|
|
62%|██████████████████████████████████████████████████████████████████████▏ | 419/681 [29:04<11:51, 2.71s/it]
62%|██████████████████████████████████████████████████████████████████████▎ | 420/681 [29:06<11:37, 2.67s/it]
{'loss': 0.6883, 'grad_norm': 52.36576843261719, 'learning_rate': 1.9401235374032425e-07, 'rewards/chosen': -0.499639630317688, 'rewards/rejected': -1.5953891277313232, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.0957496166229248, 'logps/chosen': -174.64231872558594, 'logps/rejected': -515.6880493164062, 'logps/ref_chosen': -37.82621765136719, 'logps/ref_rejected': -76.69117736816406, 'logits/chosen': -2.424182891845703, 'logits/rejected': -2.128852605819702, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0036365704145282507, 'epsilon_dpo/loss_margin_mean': 302.1807556152344, 'epsilon_dpo/beta_margin_mean': 1.0957496166229248, 'epsilon_dpo/beta_margin_std': 0.781441330909729, 'epsilon_dpo/beta_margin_grad_mean': -0.27526789903640747, 'epsilon_dpo/beta_margin_grad_std': 0.14264513552188873, 'kl/beta': 0.0036636844743043184, 'kl/avg_steps': 0.75, 'epoch': 0.62}
|
|||
|
|
62%|██████████████████████████████████████████████████████████████████████▎ | 420/681 [29:07<11:37, 2.67s/it]
62%|██████████████████████████████████████████████████████████████████████▍ | 421/681 [29:09<11:20, 2.62s/it]
{'loss': 0.6841, 'grad_norm': 42.273597717285156, 'learning_rate': 1.9276236251246653e-07, 'rewards/chosen': -0.49425339698791504, 'rewards/rejected': -1.5823595523834229, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.0881061553955078, 'logps/chosen': -170.40980529785156, 'logps/rejected': -533.166748046875, 'logps/ref_chosen': -33.550575256347656, 'logps/ref_rejected': -93.99878692626953, 'logits/chosen': -2.356675863265991, 'logits/rejected': -2.3096842765808105, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.003604953410103917, 'epsilon_dpo/loss_margin_mean': 302.30877685546875, 'epsilon_dpo/beta_margin_mean': 1.0881061553955078, 'epsilon_dpo/beta_margin_std': 0.751617968082428, 'epsilon_dpo/beta_margin_grad_mean': -0.27535223960876465, 'epsilon_dpo/beta_margin_grad_std': 0.13682620227336884, 'kl/beta': 0.003636411391198635, 'kl/avg_steps': 0.875, 'epoch': 0.62}
|
|||
|
|
62%|██████████████████████████████████████████████████████████████████████▍ | 421/681 [29:09<11:20, 2.62s/it]
62%|██████████████████████████████████████████████████████████████████████▋ | 422/681 [29:11<11:10, 2.59s/it]
{'loss': 0.7896, 'grad_norm': 46.02511215209961, 'learning_rate': 1.9151387954958792e-07, 'rewards/chosen': -0.6386697292327881, 'rewards/rejected': -1.6290339231491089, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.9903641939163208, 'logps/chosen': -231.19760131835938, 'logps/rejected': -547.3240356445312, 'logps/ref_chosen': -53.665977478027344, 'logps/ref_rejected': -91.93289947509766, 'logits/chosen': -2.373016357421875, 'logits/rejected': -2.3339807987213135, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0035804433282464743, 'epsilon_dpo/loss_margin_mean': 277.8594970703125, 'epsilon_dpo/beta_margin_mean': 0.9903641939163208, 'epsilon_dpo/beta_margin_std': 0.89506596326828, 'epsilon_dpo/beta_margin_grad_mean': -0.29801541566848755, 'epsilon_dpo/beta_margin_grad_std': 0.17275013029575348, 'kl/beta': 0.0036048688925802708, 'kl/avg_steps': 0.6875, 'epoch': 0.62}
|
|||
|
|
62%|██████████████████████████████████████████████████████████████████████▋ | 422/681 [29:12<11:10, 2.59s/it]
62%|██████████████████████████████████████████████████████████████████████▊ | 423/681 [29:14<10:50, 2.52s/it]
{'loss': 0.6702, 'grad_norm': 43.717838287353516, 'learning_rate': 1.902669377503756e-07, 'rewards/chosen': -0.5520890951156616, 'rewards/rejected': -1.6618356704711914, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.1097465753555298, 'logps/chosen': -190.4375762939453, 'logps/rejected': -560.4072265625, 'logps/ref_chosen': -35.00245666503906, 'logps/ref_rejected': -91.82589721679688, 'logits/chosen': -2.299365758895874, 'logits/rejected': -2.2865123748779297, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.003549282206222415, 'epsilon_dpo/loss_margin_mean': 313.14617919921875, 'epsilon_dpo/beta_margin_mean': 1.1097465753555298, 'epsilon_dpo/beta_margin_std': 0.7358165383338928, 'epsilon_dpo/beta_margin_grad_mean': -0.2701137661933899, 'epsilon_dpo/beta_margin_grad_std': 0.13685345649719238, 'kl/beta': 0.003580254502594471, 'kl/avg_steps': 0.875, 'epoch': 0.62}
|
|||
|
|
62%|██████████████████████████████████████████████████████████████████████▊ | 423/681 [29:14<10:50, 2.52s/it]
62%|██████████████████████████████████████████████████████████████████████▉ | 424/681 [29:16<10:52, 2.54s/it]
{'loss': 0.7749, 'grad_norm': 42.60936737060547, 'learning_rate': 1.890215699729057e-07, 'rewards/chosen': -0.543306291103363, 'rewards/rejected': -1.4932714700698853, 'rewards/accuracies': 0.875, 'rewards/margins': 0.9499651789665222, 'logps/chosen': -187.34298706054688, 'logps/rejected': -494.61981201171875, 'logps/ref_chosen': -33.510772705078125, 'logps/ref_rejected': -70.15070343017578, 'logits/chosen': -2.3963804244995117, 'logits/rejected': -2.1616666316986084, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0035229322966188192, 'epsilon_dpo/loss_margin_mean': 270.63690185546875, 'epsilon_dpo/beta_margin_mean': 0.9499651789665222, 'epsilon_dpo/beta_margin_std': 0.7828956842422485, 'epsilon_dpo/beta_margin_grad_mean': -0.30060890316963196, 'epsilon_dpo/beta_margin_grad_std': 0.15238063037395477, 'kl/beta': 0.0035491990856826305, 'kl/avg_steps': 0.75, 'epoch': 0.62}
|
|||
|
|
62%|██████████████████████████████████████████████████████████████████████▉ | 424/681 [29:16<10:52, 2.54s/it]
62%|███████████████████████████████████████████████████████████████████████▏ | 425/681 [29:19<10:43, 2.51s/it]
{'loss': 0.7154, 'grad_norm': 47.06675338745117, 'learning_rate': 1.8777780903377732e-07, 'rewards/chosen': -0.47508615255355835, 'rewards/rejected': -1.5563328266143799, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.0812466144561768, 'logps/chosen': -167.17559814453125, 'logps/rejected': -549.517333984375, 'logps/ref_chosen': -31.619510650634766, 'logps/ref_rejected': -103.71711730957031, 'logits/chosen': -2.3238930702209473, 'logits/rejected': -2.3823156356811523, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0034945050720125437, 'epsilon_dpo/loss_margin_mean': 310.2441101074219, 'epsilon_dpo/beta_margin_mean': 1.0812467336654663, 'epsilon_dpo/beta_margin_std': 0.8364912867546082, 'epsilon_dpo/beta_margin_grad_mean': -0.27929848432540894, 'epsilon_dpo/beta_margin_grad_std': 0.15515829622745514, 'kl/beta': 0.0035227781627327204, 'kl/avg_steps': 0.8125, 'epoch': 0.62}
|
|||
|
|
62%|███████████████████████████████████████████████████████████████████████▏ | 425/681 [29:19<10:43, 2.51s/it]
63%|███████████████████████████████████████████████████████████████████████▎ | 426/681 [29:21<10:34, 2.49s/it]
{'loss': 0.7381, 'grad_norm': 66.66638946533203, 'learning_rate': 1.8653568770724803e-07, 'rewards/chosen': -0.48144960403442383, 'rewards/rejected': -1.4888947010040283, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.007444977760315, 'logps/chosen': -187.71844482421875, 'logps/rejected': -515.9197387695312, 'logps/ref_chosen': -49.42609405517578, 'logps/ref_rejected': -86.20869445800781, 'logits/chosen': -2.42264986038208, 'logits/rejected': -2.2819976806640625, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.003468525130301714, 'epsilon_dpo/loss_margin_mean': 291.418701171875, 'epsilon_dpo/beta_margin_mean': 1.007444977760315, 'epsilon_dpo/beta_margin_std': 0.7751356959342957, 'epsilon_dpo/beta_margin_grad_mean': -0.28968408703804016, 'epsilon_dpo/beta_margin_grad_std': 0.1473844200372696, 'kl/beta': 0.00349438632838428, 'kl/avg_steps': 0.75, 'epoch': 0.63}
|
|||
|
|
63%|███████████████████████████████████████████████████████████████████████▎ | 426/681 [29:21<10:34, 2.49s/it]
63%|███████████████████████████████████████████████████████████████████████▍ | 427/681 [29:24<10:30, 2.48s/it]
{'loss': 0.8242, 'grad_norm': 60.45020294189453, 'learning_rate': 1.8529523872436977e-07, 'rewards/chosen': -0.5330853462219238, 'rewards/rejected': -1.3945105075836182, 'rewards/accuracies': 0.875, 'rewards/margins': 0.8614251613616943, 'logps/chosen': -194.640869140625, 'logps/rejected': -489.7284240722656, 'logps/ref_chosen': -40.53302001953125, 'logps/ref_rejected': -84.44095611572266, 'logits/chosen': -2.4879813194274902, 'logits/rejected': -2.38423490524292, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0034443254116922617, 'epsilon_dpo/loss_margin_mean': 251.1796112060547, 'epsilon_dpo/beta_margin_mean': 0.8614251017570496, 'epsilon_dpo/beta_margin_std': 0.760391354560852, 'epsilon_dpo/beta_margin_grad_mean': -0.3163089156150818, 'epsilon_dpo/beta_margin_grad_std': 0.1538737267255783, 'kl/beta': 0.0034683735575526953, 'kl/avg_steps': 0.703125, 'epoch': 0.63}
|
|||
|
|
63%|███████████████████████████████████████████████████████████████████████▍ | 427/681 [29:24<10:30, 2.48s/it]
63%|███████████████████████████████████████████████████████████████████████▋ | 428/681 [29:26<10:40, 2.53s/it]
{'loss': 0.7038, 'grad_norm': 48.36793899536133, 'learning_rate': 1.8405649477212697e-07, 'rewards/chosen': -0.5489051938056946, 'rewards/rejected': -1.6179664134979248, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.069061040878296, 'logps/chosen': -196.82138061523438, 'logps/rejected': -585.9830322265625, 'logps/ref_chosen': -36.43036651611328, 'logps/ref_rejected': -111.91488647460938, 'logits/chosen': -2.412050724029541, 'logits/rejected': -2.416749954223633, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0034143617376685143, 'epsilon_dpo/loss_margin_mean': 313.6771545410156, 'epsilon_dpo/beta_margin_mean': 1.0690611600875854, 'epsilon_dpo/beta_margin_std': 0.7979580163955688, 'epsilon_dpo/beta_margin_grad_mean': -0.2803729176521301, 'epsilon_dpo/beta_margin_grad_std': 0.14126239717006683, 'kl/beta': 0.0034441568423062563, 'kl/avg_steps': 0.875, 'epoch': 0.63}
|
|||
|
|
63%|███████████████████████████████████████████████████████████████████████▋ | 428/681 [29:26<10:40, 2.53s/it]
63%|███████████████████████████████████████████████████████████████████████▊ | 429/681 [29:29<10:43, 2.55s/it]
{'loss': 0.6748, 'grad_norm': 45.01988220214844, 'learning_rate': 1.828194884925749e-07, 'rewards/chosen': -0.5519835352897644, 'rewards/rejected': -1.6421988010406494, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.0902152061462402, 'logps/chosen': -211.84674072265625, 'logps/rejected': -583.9923095703125, 'logps/ref_chosen': -49.30812072753906, 'logps/ref_rejected': -98.94145965576172, 'logits/chosen': -2.5120902061462402, 'logits/rejected': -2.3635692596435547, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0033868795726448298, 'epsilon_dpo/loss_margin_mean': 322.5122375488281, 'epsilon_dpo/beta_margin_mean': 1.0902152061462402, 'epsilon_dpo/beta_margin_std': 0.7191058397293091, 'epsilon_dpo/beta_margin_grad_mean': -0.2734263241291046, 'epsilon_dpo/beta_margin_grad_std': 0.13189740478992462, 'kl/beta': 0.003414281876757741, 'kl/avg_steps': 0.8125, 'epoch': 0.63}
|
|||
|
|
63%|███████████████████████████████████████████████████████████████████████▊ | 429/681 [29:29<10:43, 2.55s/it]
63%|███████████████████████████████████████████████████████████████████████▉ | 430/681 [29:32<11:02, 2.64s/it]
{'loss': 0.7561, 'grad_norm': 43.82745361328125, 'learning_rate': 1.8158425248197928e-07, 'rewards/chosen': -0.5032367706298828, 'rewards/rejected': -1.4752440452575684, 'rewards/accuracies': 0.875, 'rewards/margins': 0.9720072746276855, 'logps/chosen': -194.5563507080078, 'logps/rejected': -549.539306640625, 'logps/ref_chosen': -45.3841438293457, 'logps/ref_rejected': -110.27545928955078, 'logits/chosen': -2.4356656074523926, 'logits/rejected': -2.5032784938812256, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0033638167660683393, 'epsilon_dpo/loss_margin_mean': 290.09161376953125, 'epsilon_dpo/beta_margin_mean': 0.9720072150230408, 'epsilon_dpo/beta_margin_std': 0.7672375440597534, 'epsilon_dpo/beta_margin_grad_mean': -0.2967393100261688, 'epsilon_dpo/beta_margin_grad_std': 0.14850302040576935, 'kl/beta': 0.003386764321476221, 'kl/avg_steps': 0.6875, 'epoch': 0.63}
|
|||
|
|
63%|███████████████████████████████████████████████████████████████████████▉ | 430/681 [29:32<11:02, 2.64s/it]
63%|████████████████████████████████████████████████████████████████████████▏ | 431/681 [29:34<10:43, 2.57s/it]
{'loss': 0.6656, 'grad_norm': 37.97023391723633, 'learning_rate': 1.8035081928995788e-07, 'rewards/chosen': -0.4194945991039276, 'rewards/rejected': -1.5508911609649658, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.1313966512680054, 'logps/chosen': -159.66354370117188, 'logps/rejected': -563.6588134765625, 'logps/ref_chosen': -34.30770492553711, 'logps/ref_rejected': -98.43756866455078, 'logits/chosen': -2.4460361003875732, 'logits/rejected': -2.4488284587860107, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.00333664333447814, 'epsilon_dpo/loss_margin_mean': 339.86541748046875, 'epsilon_dpo/beta_margin_mean': 1.1313966512680054, 'epsilon_dpo/beta_margin_std': 0.7779242396354675, 'epsilon_dpo/beta_margin_grad_mean': -0.2679252624511719, 'epsilon_dpo/beta_margin_grad_std': 0.13725493848323822, 'kl/beta': 0.003363639349117875, 'kl/avg_steps': 0.8125, 'epoch': 0.63}
|
|||
|
|
63%|████████████████████████████████████████████████████████████████████████▏ | 431/681 [29:34<10:43, 2.57s/it]
63%|████████████████████████████████████████████████████████████████████████▎ | 432/681 [29:37<10:46, 2.60s/it]
{'loss': 0.7004, 'grad_norm': 44.22251892089844, 'learning_rate': 1.791192214186223e-07, 'rewards/chosen': -0.4821677803993225, 'rewards/rejected': -1.5151182413101196, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.0329504013061523, 'logps/chosen': -184.24755859375, 'logps/rejected': -563.4796142578125, 'logps/ref_chosen': -38.85405349731445, 'logps/ref_rejected': -105.27049255371094, 'logits/chosen': -2.466712236404419, 'logits/rejected': -2.4444732666015625, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0033087090123444796, 'epsilon_dpo/loss_margin_mean': 312.8155822753906, 'epsilon_dpo/beta_margin_mean': 1.032950520515442, 'epsilon_dpo/beta_margin_std': 0.699227511882782, 'epsilon_dpo/beta_margin_grad_mean': -0.28203877806663513, 'epsilon_dpo/beta_margin_grad_std': 0.12971222400665283, 'kl/beta': 0.003336530178785324, 'kl/avg_steps': 0.84375, 'epoch': 0.63}
|
|||
|
|
63%|████████████████████████████████████████████████████████████████████████▎ | 432/681 [29:37<10:46, 2.60s/it]
64%|████████████████████████████████████████████████████████████████████████▍ | 433/681 [29:39<10:34, 2.56s/it]
{'loss': 0.6991, 'grad_norm': 45.43602752685547, 'learning_rate': 1.7788949132172193e-07, 'rewards/chosen': -0.5854209661483765, 'rewards/rejected': -1.6637802124023438, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.0783591270446777, 'logps/chosen': -213.7806396484375, 'logps/rejected': -610.318359375, 'logps/ref_chosen': -35.81452178955078, 'logps/ref_rejected': -103.11997985839844, 'logits/chosen': -2.3216044902801514, 'logits/rejected': -2.3884963989257812, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0032830932177603245, 'epsilon_dpo/loss_margin_mean': 329.2322692871094, 'epsilon_dpo/beta_margin_mean': 1.0783592462539673, 'epsilon_dpo/beta_margin_std': 0.7789259552955627, 'epsilon_dpo/beta_margin_grad_mean': -0.2790597379207611, 'epsilon_dpo/beta_margin_grad_std': 0.14551861584186554, 'kl/beta': 0.0033086135517805815, 'kl/avg_steps': 0.78125, 'epoch': 0.64}
|
|||
|
|
64%|████████████████████████████████████████████████████████████████████████▍ | 433/681 [29:39<10:34, 2.56s/it]
64%|████████████████████████████████████████████████████████████████████████▋ | 434/681 [29:42<10:25, 2.53s/it]
{'loss': 0.7919, 'grad_norm': 43.43961715698242, 'learning_rate': 1.7666166140378853e-07, 'rewards/chosen': -0.6385765075683594, 'rewards/rejected': -1.5220519304275513, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.8834754228591919, 'logps/chosen': -245.33560180664062, 'logps/rejected': -550.40966796875, 'logps/ref_chosen': -49.93016052246094, 'logps/ref_rejected': -83.06277465820312, 'logits/chosen': -2.4602980613708496, 'logits/rejected': -2.3824265003204346, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.00325866905041039, 'epsilon_dpo/loss_margin_mean': 271.9414367675781, 'epsilon_dpo/beta_margin_mean': 0.8834754228591919, 'epsilon_dpo/beta_margin_std': 0.7322390079498291, 'epsilon_dpo/beta_margin_grad_mean': -0.31264179944992065, 'epsilon_dpo/beta_margin_grad_std': 0.1338089406490326, 'kl/beta': 0.0032829653937369585, 'kl/avg_steps': 0.75, 'epoch': 0.64}
|
|||
|
|
64%|████████████████████████████████████████████████████████████████████████▋ | 434/681 [29:42<10:25, 2.53s/it]
64%|████████████████████████████████████████████████████████████████████████▊ | 435/681 [29:44<10:07, 2.47s/it]
{'loss': 0.6928, 'grad_norm': 42.341400146484375, 'learning_rate': 1.7543576401928218e-07, 'rewards/chosen': -0.4249953627586365, 'rewards/rejected': -1.4922096729278564, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.0672142505645752, 'logps/chosen': -165.0016326904297, 'logps/rejected': -555.97021484375, 'logps/ref_chosen': -33.67361831665039, 'logps/ref_rejected': -93.73330688476562, 'logits/chosen': -2.4340808391571045, 'logits/rejected': -2.4357552528381348, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.003229319117963314, 'epsilon_dpo/loss_margin_mean': 330.9088439941406, 'epsilon_dpo/beta_margin_mean': 1.0672143697738647, 'epsilon_dpo/beta_margin_std': 0.7533097863197327, 'epsilon_dpo/beta_margin_grad_mean': -0.2789570689201355, 'epsilon_dpo/beta_margin_grad_std': 0.13298888504505157, 'kl/beta': 0.0032585265580564737, 'kl/avg_steps': 0.90625, 'epoch': 0.64}
|
|||
|
|
64%|████████████████████████████████████████████████████████████████████████▊ | 435/681 [29:44<10:07, 2.47s/it]
64%|████████████████████████████████████████████████████████████████████████▉ | 436/681 [29:47<10:17, 2.52s/it]
{'loss': 0.7008, 'grad_norm': 51.16184616088867, 'learning_rate': 1.742118314717391e-07, 'rewards/chosen': -0.4683375954627991, 'rewards/rejected': -1.468498945236206, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.0001612901687622, 'logps/chosen': -188.12747192382812, 'logps/rejected': -547.4158325195312, 'logps/ref_chosen': -42.048667907714844, 'logps/ref_rejected': -88.41279602050781, 'logits/chosen': -2.5231666564941406, 'logits/rejected': -2.4640743732452393, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0032013256568461657, 'epsilon_dpo/loss_margin_mean': 312.92425537109375, 'epsilon_dpo/beta_margin_mean': 1.0001612901687622, 'epsilon_dpo/beta_margin_std': 0.6187871694564819, 'epsilon_dpo/beta_margin_grad_mean': -0.2841736078262329, 'epsilon_dpo/beta_margin_grad_std': 0.11989546567201614, 'kl/beta': 0.003229261375963688, 'kl/avg_steps': 0.875, 'epoch': 0.64}
|
|||
|
|
64%|████████████████████████████████████████████████████████████████████████▉ | 436/681 [29:47<10:17, 2.52s/it]
64%|█████████████████████████████████████████████████████████████████████████▏ | 437/681 [29:49<10:15, 2.52s/it]
{'loss': 0.7736, 'grad_norm': 45.89207458496094, 'learning_rate': 1.7298989601292036e-07, 'rewards/chosen': -0.5791641473770142, 'rewards/rejected': -1.4944398403167725, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.9152756929397583, 'logps/chosen': -226.48953247070312, 'logps/rejected': -557.265625, 'logps/ref_chosen': -44.77692413330078, 'logps/ref_rejected': -86.48928833007812, 'logits/chosen': -2.4697906970977783, 'logits/rejected': -2.390430450439453, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0031775590032339096, 'epsilon_dpo/loss_margin_mean': 289.0637512207031, 'epsilon_dpo/beta_margin_mean': 0.9152756929397583, 'epsilon_dpo/beta_margin_std': 0.6979488134384155, 'epsilon_dpo/beta_margin_grad_mean': -0.30295178294181824, 'epsilon_dpo/beta_margin_grad_std': 0.14245736598968506, 'kl/beta': 0.0032012504525482655, 'kl/avg_steps': 0.75, 'epoch': 0.64}
|
|||
|
|
64%|█████████████████████████████████████████████████████████████████████████▏ | 437/681 [29:49<10:15, 2.52s/it]
64%|█████████████████████████████████████████████████████████████████████████▎ | 438/681 [29:52<09:59, 2.47s/it]
{'loss': 0.6943, 'grad_norm': 40.396175384521484, 'learning_rate': 1.7176998984196144e-07, 'rewards/chosen': -0.41470038890838623, 'rewards/rejected': -1.4498802423477173, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.035179853439331, 'logps/chosen': -164.937744140625, 'logps/rejected': -549.906005859375, 'logps/ref_chosen': -33.662109375, 'logps/ref_rejected': -89.52166748046875, 'logits/chosen': -2.473397731781006, 'logits/rejected': -2.3974690437316895, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0031519182957708836, 'epsilon_dpo/loss_margin_mean': 329.1087341308594, 'epsilon_dpo/beta_margin_mean': 1.035179853439331, 'epsilon_dpo/beta_margin_std': 0.6745886206626892, 'epsilon_dpo/beta_margin_grad_mean': -0.2808303236961365, 'epsilon_dpo/beta_margin_grad_std': 0.12779134511947632, 'kl/beta': 0.0031774197705090046, 'kl/avg_steps': 0.8125, 'epoch': 0.64}
|
|||
|
|
64%|█████████████████████████████████████████████████████████████████████████▎ | 438/681 [29:52<09:59, 2.47s/it]
64%|█████████████████████████████████████████████████████████████████████████▍ | 439/681 [29:54<10:06, 2.50s/it]
{'loss': 0.7563, 'grad_norm': 45.90419006347656, 'learning_rate': 1.7055214510452458e-07, 'rewards/chosen': -0.5056626200675964, 'rewards/rejected': -1.4195419549942017, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.91387939453125, 'logps/chosen': -196.35198974609375, 'logps/rejected': -541.820556640625, 'logps/ref_chosen': -34.986392974853516, 'logps/ref_rejected': -87.497314453125, 'logits/chosen': -2.472698926925659, 'logits/rejected': -2.5124192237854004, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.003126515308395028, 'epsilon_dpo/loss_margin_mean': 292.9576721191406, 'epsilon_dpo/beta_margin_mean': 0.91387939453125, 'epsilon_dpo/beta_margin_std': 0.6512243747711182, 'epsilon_dpo/beta_margin_grad_mean': -0.3024454414844513, 'epsilon_dpo/beta_margin_grad_std': 0.1240093857049942, 'kl/beta': 0.0031518111936748028, 'kl/avg_steps': 0.8125, 'epoch': 0.64}
|
|||
|
|
64%|█████████████████████████████████████████████████████████████████████████▍ | 439/681 [29:54<10:06, 2.50s/it]
65%|█████████████████████████████████████████████████████████████████████████▋ | 440/681 [29:57<10:18, 2.56s/it]
{'loss': 0.761, 'grad_norm': 51.92120361328125, 'learning_rate': 1.6933639389195134e-07, 'rewards/chosen': -0.5756343603134155, 'rewards/rejected': -1.509265661239624, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.9336312413215637, 'logps/chosen': -238.4813232421875, 'logps/rejected': -591.0408935546875, 'logps/ref_chosen': -53.56586837768555, 'logps/ref_rejected': -104.3643569946289, 'logits/chosen': -2.5072128772735596, 'logits/rejected': -2.5406949520111084, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.003103271359577775, 'epsilon_dpo/loss_margin_mean': 301.76104736328125, 'epsilon_dpo/beta_margin_mean': 0.9336312413215637, 'epsilon_dpo/beta_margin_std': 0.7152769565582275, 'epsilon_dpo/beta_margin_grad_mean': -0.3023149073123932, 'epsilon_dpo/beta_margin_grad_std': 0.134578138589859, 'kl/beta': 0.003126409137621522, 'kl/avg_steps': 0.75, 'epoch': 0.65}
|
|||
|
|
65%|█████████████████████████████████████████████████████████████████████████▋ | 440/681 [29:57<10:18, 2.56s/it]
65%|█████████████████████████████████████████████████████████████████████████▊ | 441/681 [30:00<10:35, 2.65s/it]
{'loss': 0.7438, 'grad_norm': 40.502708435058594, 'learning_rate': 1.681227682404166e-07, 'rewards/chosen': -0.49797898530960083, 'rewards/rejected': -1.4459290504455566, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.947950005531311, 'logps/chosen': -200.94158935546875, 'logps/rejected': -573.1942138671875, 'logps/ref_chosen': -39.209449768066406, 'logps/ref_rejected': -102.78851318359375, 'logits/chosen': -2.5055270195007324, 'logits/rejected': -2.469489812850952, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.0030753209721297026, 'epsilon_dpo/loss_margin_mean': 308.67352294921875, 'epsilon_dpo/beta_margin_mean': 0.9479500651359558, 'epsilon_dpo/beta_margin_std': 0.6735605597496033, 'epsilon_dpo/beta_margin_grad_mean': -0.2959996163845062, 'epsilon_dpo/beta_margin_grad_std': 0.12961319088935852, 'kl/beta': 0.003103135619312525, 'kl/avg_steps': 0.90625, 'epoch': 0.65}
|
|||
|
|
65%|█████████████████████████████████████████████████████████████████████████▊ | 441/681 [30:00<10:35, 2.65s/it]
65%|█████████████████████████████████████████████████████████████████████████▉ | 442/681 [30:02<10:11, 2.56s/it]
{'loss': 0.675, 'grad_norm': 37.710105895996094, 'learning_rate': 1.669113001300851e-07, 'rewards/chosen': -0.39891505241394043, 'rewards/rejected': -1.444936990737915, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.046021819114685, 'logps/chosen': -159.370849609375, 'logps/rejected': -557.7063598632812, 'logps/ref_chosen': -29.0069580078125, 'logps/ref_rejected': -83.71453857421875, 'logits/chosen': -2.491802930831909, 'logits/rejected': -2.457737922668457, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.003050584578886628, 'epsilon_dpo/loss_margin_mean': 343.6279602050781, 'epsilon_dpo/beta_margin_mean': 1.046021819114685, 'epsilon_dpo/beta_margin_std': 0.6180484294891357, 'epsilon_dpo/beta_margin_grad_mean': -0.2755725681781769, 'epsilon_dpo/beta_margin_grad_std': 0.11769750714302063, 'kl/beta': 0.0030752660240978003, 'kl/avg_steps': 0.8125, 'epoch': 0.65}
|
|||
|
|
65%|█████████████████████████████████████████████████████████████████████████▉ | 442/681 [30:02<10:11, 2.56s/it]
65%|██████████████████████████████████████████████████████████████████████████▏ | 443/681 [30:05<09:59, 2.52s/it]
{'loss': 0.841, 'grad_norm': 46.755645751953125, 'learning_rate': 1.6570202148426815e-07, 'rewards/chosen': -0.549048900604248, 'rewards/rejected': -1.3913331031799316, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.8422842025756836, 'logps/chosen': -233.57965087890625, 'logps/rejected': -553.4365234375, 'logps/ref_chosen': -52.68767166137695, 'logps/ref_rejected': -93.39274597167969, 'logits/chosen': -2.5168378353118896, 'logits/rejected': -2.499556541442871, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0030279052443802357, 'epsilon_dpo/loss_margin_mean': 279.1518249511719, 'epsilon_dpo/beta_margin_mean': 0.8422841429710388, 'epsilon_dpo/beta_margin_std': 0.7962964177131653, 'epsilon_dpo/beta_margin_grad_mean': -0.3231920897960663, 'epsilon_dpo/beta_margin_grad_std': 0.15162836015224457, 'kl/beta': 0.003050480969250202, 'kl/avg_steps': 0.75, 'epoch': 0.65}
|
|||
|
|
65%|██████████████████████████████████████████████████████████████████████████▏ | 443/681 [30:05<09:59, 2.52s/it]
65%|██████████████████████████████████████████████████████████████████████████▎ | 444/681 [30:07<09:58, 2.52s/it]
{'loss': 0.6502, 'grad_norm': 38.41836929321289, 'learning_rate': 1.6449496416858282e-07, 'rewards/chosen': -0.4290057420730591, 'rewards/rejected': -1.5417864322662354, 'rewards/accuracies': 0.984375, 'rewards/margins': 1.1127806901931763, 'logps/chosen': -178.10812377929688, 'logps/rejected': -618.6232299804688, 'logps/ref_chosen': -35.20741271972656, 'logps/ref_rejected': -104.47367858886719, 'logits/chosen': -2.462678909301758, 'logits/rejected': -2.5427470207214355, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.0029996871016919613, 'epsilon_dpo/loss_margin_mean': 371.2488708496094, 'epsilon_dpo/beta_margin_mean': 1.1127806901931763, 'epsilon_dpo/beta_margin_std': 0.6654544472694397, 'epsilon_dpo/beta_margin_grad_mean': -0.26602280139923096, 'epsilon_dpo/beta_margin_grad_std': 0.12350434064865112, 'kl/beta': 0.003027772530913353, 'kl/avg_steps': 0.9375, 'epoch': 0.65}
|
|||
|
|
65%|██████████████████████████████████████████████████████████████████████████▎ | 444/681 [30:07<09:58, 2.52s/it]
65%|██████████████████████████████████████████████████████████████████████████▍ | 445/681 [30:10<09:59, 2.54s/it]
{'loss': 0.8334, 'grad_norm': 48.016639709472656, 'learning_rate': 1.6329015999011182e-07, 'rewards/chosen': -0.5069864392280579, 'rewards/rejected': -1.3884000778198242, 'rewards/accuracies': 0.875, 'rewards/margins': 0.8814135789871216, 'logps/chosen': -216.25303649902344, 'logps/rejected': -565.91015625, 'logps/ref_chosen': -46.78347396850586, 'logps/ref_rejected': -99.2047119140625, 'logits/chosen': -2.5741279125213623, 'logits/rejected': -2.5458946228027344, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0029783889185637236, 'epsilon_dpo/loss_margin_mean': 297.23583984375, 'epsilon_dpo/beta_margin_mean': 0.8814136385917664, 'epsilon_dpo/beta_margin_std': 0.8328824639320374, 'epsilon_dpo/beta_margin_grad_mean': -0.31596365571022034, 'epsilon_dpo/beta_margin_grad_std': 0.1648440957069397, 'kl/beta': 0.002999651012942195, 'kl/avg_steps': 0.71875, 'epoch': 0.65}
|
|||
|
|
65%|██████████████████████████████████████████████████████████████████████████▍ | 445/681 [30:10<09:59, 2.54s/it]
65%|██████████████████████████████████████████████████████████████████████████▋ | 446/681 [30:12<10:00, 2.55s/it]
{'loss': 0.6534, 'grad_norm': 50.814979553222656, 'learning_rate': 1.6208764069656578e-07, 'rewards/chosen': -0.37178486585617065, 'rewards/rejected': -1.4675066471099854, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.0957218408584595, 'logps/chosen': -160.44252014160156, 'logps/rejected': -607.453857421875, 'logps/ref_chosen': -34.56015396118164, 'logps/ref_rejected': -109.97004699707031, 'logits/chosen': -2.528761863708496, 'logits/rejected': -2.6162219047546387, 'kl/p_epsilon_steps': 0.96875, 'kl/n_epsilon_steps': 0.03125, 'epsilon_dpo/beta': 0.0029506187420338392, 'epsilon_dpo/loss_margin_mean': 371.6014099121094, 'epsilon_dpo/beta_margin_mean': 1.0957218408584595, 'epsilon_dpo/beta_margin_std': 0.6562060713768005, 'epsilon_dpo/beta_margin_grad_mean': -0.2692401707172394, 'epsilon_dpo/beta_margin_grad_std': 0.11489293724298477, 'kl/beta': 0.002978244796395302, 'kl/avg_steps': 0.9375, 'epoch': 0.65}
|
|||
|
|
65%|██████████████████████████████████████████████████████████████████████████▋ | 446/681 [30:12<10:00, 2.55s/it]
66%|██████████████████████████████████████████████████████████████████████████▊ | 447/681 [30:15<09:45, 2.50s/it]
{'loss': 0.7578, 'grad_norm': 40.274986267089844, 'learning_rate': 1.608874379754465e-07, 'rewards/chosen': -0.471713662147522, 'rewards/rejected': -1.4239914417266846, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.9522777795791626, 'logps/chosen': -199.97299194335938, 'logps/rejected': -592.6107788085938, 'logps/ref_chosen': -39.49730682373047, 'logps/ref_rejected': -105.97085571289062, 'logits/chosen': -2.502809762954712, 'logits/rejected': -2.5910682678222656, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0029278243891894817, 'epsilon_dpo/loss_margin_mean': 326.1642150878906, 'epsilon_dpo/beta_margin_mean': 0.9522778391838074, 'epsilon_dpo/beta_margin_std': 0.7474956512451172, 'epsilon_dpo/beta_margin_grad_mean': -0.2999436855316162, 'epsilon_dpo/beta_margin_grad_std': 0.13803143799304962, 'kl/beta': 0.00295058311894536, 'kl/avg_steps': 0.78125, 'epoch': 0.66}
|
|||
|
|
66%|██████████████████████████████████████████████████████████████████████████▊ | 447/681 [30:15<09:45, 2.50s/it]
66%|██████████████████████████████████████████████████████████████████████████▉ | 448/681 [30:17<09:53, 2.55s/it]
{'loss': 0.7027, 'grad_norm': 41.37987518310547, 'learning_rate': 1.5968958345321177e-07, 'rewards/chosen': -0.5373802781105042, 'rewards/rejected': -1.5529210567474365, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.0155407190322876, 'logps/chosen': -227.657958984375, 'logps/rejected': -644.6298217773438, 'logps/ref_chosen': -42.827239990234375, 'logps/ref_rejected': -109.36424255371094, 'logits/chosen': -2.5615386962890625, 'logits/rejected': -2.621204376220703, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.0029014681931585073, 'epsilon_dpo/loss_margin_mean': 350.43487548828125, 'epsilon_dpo/beta_margin_mean': 1.0155407190322876, 'epsilon_dpo/beta_margin_std': 0.674639880657196, 'epsilon_dpo/beta_margin_grad_mean': -0.2846612334251404, 'epsilon_dpo/beta_margin_grad_std': 0.12346479296684265, 'kl/beta': 0.0029277103021740913, 'kl/avg_steps': 0.90625, 'epoch': 0.66}
|
|||
|
|
66%|██████████████████████████████████████████████████████████████████████████▉ | 448/681 [30:17<09:53, 2.55s/it]
66%|███████████████████████████████████████████████████████████████████████████▏ | 449/681 [30:20<09:36, 2.49s/it]
{'loss': 0.7298, 'grad_norm': 43.268863677978516, 'learning_rate': 1.584941086944423e-07, 'rewards/chosen': -0.4556068181991577, 'rewards/rejected': -1.4396498203277588, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.9840430021286011, 'logps/chosen': -194.70394897460938, 'logps/rejected': -596.5521240234375, 'logps/ref_chosen': -36.90496826171875, 'logps/ref_rejected': -95.95344543457031, 'logits/chosen': -2.5850870609283447, 'logits/rejected': -2.595974922180176, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.002878130180761218, 'epsilon_dpo/loss_margin_mean': 342.7997131347656, 'epsilon_dpo/beta_margin_mean': 0.9840430617332458, 'epsilon_dpo/beta_margin_std': 0.6940675973892212, 'epsilon_dpo/beta_margin_grad_mean': -0.28992605209350586, 'epsilon_dpo/beta_margin_grad_std': 0.13492745161056519, 'kl/beta': 0.0029014162719249725, 'kl/avg_steps': 0.8125, 'epoch': 0.66}
|
|||
|
|
66%|███████████████████████████████████████████████████████████████████████████▏ | 449/681 [30:20<09:36, 2.49s/it]
66%|███████████████████████████████████████████████████████████████████████████▎ | 450/681 [30:22<09:34, 2.49s/it]
{'loss': 0.6252, 'grad_norm': 37.674678802490234, 'learning_rate': 1.573010452010098e-07, 'rewards/chosen': -0.40372031927108765, 'rewards/rejected': -1.5394420623779297, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.1357218027114868, 'logps/chosen': -175.88995361328125, 'logps/rejected': -648.0965576171875, 'logps/ref_chosen': -34.65415573120117, 'logps/ref_rejected': -108.24179077148438, 'logits/chosen': -2.546546459197998, 'logits/rejected': -2.698519229888916, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.002853134647011757, 'epsilon_dpo/loss_margin_mean': 398.61895751953125, 'epsilon_dpo/beta_margin_mean': 1.1357218027114868, 'epsilon_dpo/beta_margin_std': 0.607581377029419, 'epsilon_dpo/beta_margin_grad_mean': -0.2585708200931549, 'epsilon_dpo/beta_margin_grad_std': 0.11334740370512009, 'kl/beta': 0.00287803215906024, 'kl/avg_steps': 0.875, 'epoch': 0.66}
|
|||
|
|
66%|███████████████████████████████████████████████████████████████████████████▎ | 450/681 [30:22<09:34, 2.49s/it]
66%|███████████████████████████████████████████████████████████████████████████▍ | 451/681 [30:25<09:33, 2.49s/it]
{'loss': 0.8563, 'grad_norm': 46.581146240234375, 'learning_rate': 1.5611042441124687e-07, 'rewards/chosen': -0.5459603071212769, 'rewards/rejected': -1.3558356761932373, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.8098753690719604, 'logps/chosen': -234.43658447265625, 'logps/rejected': -558.2542724609375, 'logps/ref_chosen': -42.703250885009766, 'logps/ref_rejected': -79.43376159667969, 'logits/chosen': -2.5236382484436035, 'logits/rejected': -2.479279041290283, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.002834628103300929, 'epsilon_dpo/loss_margin_mean': 287.087158203125, 'epsilon_dpo/beta_margin_mean': 0.8098753690719604, 'epsilon_dpo/beta_margin_std': 0.763114869594574, 'epsilon_dpo/beta_margin_grad_mean': -0.32642942667007446, 'epsilon_dpo/beta_margin_grad_std': 0.15271244943141937, 'kl/beta': 0.0028530678246170282, 'kl/avg_steps': 0.65625, 'epoch': 0.66}
|
|||
|
|
66%|███████████████████████████████████████████████████████████████████████████▍ | 451/681 [30:25<09:33, 2.49s/it]
66%|███████████████████████████████████████████████████████████████████████████▋ | 452/681 [30:28<09:56, 2.60s/it]
{'loss': 0.7379, 'grad_norm': 40.06062316894531, 'learning_rate': 1.549222776991186e-07, 'rewards/chosen': -0.3679235577583313, 'rewards/rejected': -1.315352439880371, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.9474288821220398, 'logps/chosen': -166.42276000976562, 'logps/rejected': -570.5884399414062, 'logps/ref_chosen': -35.80718231201172, 'logps/ref_rejected': -102.27734375, 'logits/chosen': -2.5114731788635254, 'logits/rejected': -2.680518627166748, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0028108321130275726, 'epsilon_dpo/loss_margin_mean': 337.69549560546875, 'epsilon_dpo/beta_margin_mean': 0.9474288821220398, 'epsilon_dpo/beta_margin_std': 0.651831328868866, 'epsilon_dpo/beta_margin_grad_mean': -0.2964177429676056, 'epsilon_dpo/beta_margin_grad_std': 0.12497884780168533, 'kl/beta': 0.002834466751664877, 'kl/avg_steps': 0.84375, 'epoch': 0.66}
|
|||
|
|
66%|███████████████████████████████████████████████████████████████████████████▋ | 452/681 [30:28<09:56, 2.60s/it]
67%|███████████████████████████████████████████████████████████████████████████▊ | 453/681 [30:30<09:34, 2.52s/it]
{'loss': 0.7343, 'grad_norm': 39.58503723144531, 'learning_rate': 1.5373663637339584e-07, 'rewards/chosen': -0.4305250644683838, 'rewards/rejected': -1.3857460021972656, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.9552209973335266, 'logps/chosen': -195.30682373046875, 'logps/rejected': -585.105224609375, 'logps/ref_chosen': -41.37712860107422, 'logps/ref_rejected': -87.86880493164062, 'logits/chosen': -2.605818748474121, 'logits/rejected': -2.60537052154541, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.002787314122542739, 'epsilon_dpo/loss_margin_mean': 343.3067321777344, 'epsilon_dpo/beta_margin_mean': 0.9552209973335266, 'epsilon_dpo/beta_margin_std': 0.6552383899688721, 'epsilon_dpo/beta_margin_grad_mean': -0.2953357696533203, 'epsilon_dpo/beta_margin_grad_std': 0.12554973363876343, 'kl/beta': 0.002810751087963581, 'kl/avg_steps': 0.84375, 'epoch': 0.67}
|
|||
|
|
67%|███████████████████████████████████████████████████████████████████████████▊ | 453/681 [30:30<09:34, 2.52s/it]
67%|████████████████████████████████████████████████████████████████████████████ | 454/681 [30:32<09:15, 2.45s/it]
{'loss': 0.7676, 'grad_norm': 55.10441207885742, 'learning_rate': 1.5255353167683017e-07, 'rewards/chosen': -0.5324690341949463, 'rewards/rejected': -1.4844902753829956, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.9520212411880493, 'logps/chosen': -236.1055145263672, 'logps/rejected': -627.3410034179688, 'logps/ref_chosen': -44.58696746826172, 'logps/ref_rejected': -90.57184600830078, 'logits/chosen': -2.5729262828826904, 'logits/rejected': -2.6592321395874023, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0027683484368026257, 'epsilon_dpo/loss_margin_mean': 345.2506103515625, 'epsilon_dpo/beta_margin_mean': 0.9520212411880493, 'epsilon_dpo/beta_margin_std': 0.7603475451469421, 'epsilon_dpo/beta_margin_grad_mean': -0.29989561438560486, 'epsilon_dpo/beta_margin_grad_std': 0.14955060184001923, 'kl/beta': 0.0027872337959706783, 'kl/avg_steps': 0.6875, 'epoch': 0.67}
|
|||
|
|
67%|████████████████████████████████████████████████████████████████████████████ | 454/681 [30:32<09:15, 2.45s/it]
67%|████████████████████████████████████████████████████████████████████████████▏ | 455/681 [30:35<09:20, 2.48s/it]
{'loss': 0.6379, 'grad_norm': 33.066959381103516, 'learning_rate': 1.5137299478533064e-07, 'rewards/chosen': -0.37842369079589844, 'rewards/rejected': -1.5500357151031494, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.171612024307251, 'logps/chosen': -160.47991943359375, 'logps/rejected': -686.5131225585938, 'logps/ref_chosen': -22.870223999023438, 'logps/ref_rejected': -121.32386779785156, 'logits/chosen': -2.561234474182129, 'logits/rejected': -2.784088611602783, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0027442548889666796, 'epsilon_dpo/loss_margin_mean': 427.5795593261719, 'epsilon_dpo/beta_margin_mean': 1.1716121435165405, 'epsilon_dpo/beta_margin_std': 0.7572227120399475, 'epsilon_dpo/beta_margin_grad_mean': -0.26028749346733093, 'epsilon_dpo/beta_margin_grad_std': 0.12892557680606842, 'kl/beta': 0.002768202219158411, 'kl/avg_steps': 0.875, 'epoch': 0.67}
|
|||
|
|
67%|████████████████████████████████████████████████████████████████████████████▏ | 455/681 [30:35<09:20, 2.48s/it]
67%|████████████████████████████████████████████████████████████████████████████▎ | 456/681 [30:37<09:10, 2.45s/it]
{'loss': 0.7064, 'grad_norm': 35.979026794433594, 'learning_rate': 1.5019505680714232e-07, 'rewards/chosen': -0.4502452611923218, 'rewards/rejected': -1.5035851001739502, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.053339958190918, 'logps/chosen': -205.63754272460938, 'logps/rejected': -664.3036499023438, 'logps/ref_chosen': -40.844276428222656, 'logps/ref_rejected': -111.70032501220703, 'logits/chosen': -2.559840202331543, 'logits/rejected': -2.7625198364257812, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.002723881509155035, 'epsilon_dpo/loss_margin_mean': 387.81005859375, 'epsilon_dpo/beta_margin_mean': 1.053339958190918, 'epsilon_dpo/beta_margin_std': 0.7655569314956665, 'epsilon_dpo/beta_margin_grad_mean': -0.2826097905635834, 'epsilon_dpo/beta_margin_grad_std': 0.13924254477024078, 'kl/beta': 0.002744190627709031, 'kl/avg_steps': 0.75, 'epoch': 0.67}
|
|||
|
|
67%|████████████████████████████████████████████████████████████████████████████▎ | 456/681 [30:37<09:10, 2.45s/it]
67%|████████████████████████████████████████████████████████████████████████████▌ | 457/681 [30:40<09:15, 2.48s/it]
{'loss': 0.7706, 'grad_norm': 48.26573944091797, 'learning_rate': 1.4901974878202627e-07, 'rewards/chosen': -0.47887659072875977, 'rewards/rejected': -1.3794221878051758, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.900545597076416, 'logps/chosen': -215.42941284179688, 'logps/rejected': -601.1631469726562, 'logps/ref_chosen': -38.554141998291016, 'logps/ref_rejected': -90.09440612792969, 'logits/chosen': -2.6022448539733887, 'logits/rejected': -2.619676351547241, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0027019022963941097, 'epsilon_dpo/loss_margin_mean': 334.1934509277344, 'epsilon_dpo/beta_margin_mean': 0.900545597076416, 'epsilon_dpo/beta_margin_std': 0.6524316668510437, 'epsilon_dpo/beta_margin_grad_mean': -0.3032781481742859, 'epsilon_dpo/beta_margin_grad_std': 0.13411802053451538, 'kl/beta': 0.002723762532696128, 'kl/avg_steps': 0.8125, 'epoch': 0.67}
|
|||
|
|
67%|████████████████████████████████████████████████████████████████████████████▌ | 457/681 [30:40<09:15, 2.48s/it]
67%|████████████████████████████████████████████████████████████████████████████▋ | 458/681 [30:42<09:09, 2.46s/it]
{'loss': 0.732, 'grad_norm': 49.126129150390625, 'learning_rate': 1.4784710168044212e-07, 'rewards/chosen': -0.4552924931049347, 'rewards/rejected': -1.4071264266967773, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.951833963394165, 'logps/chosen': -206.733642578125, 'logps/rejected': -629.8433227539062, 'logps/ref_chosen': -37.41191482543945, 'logps/ref_rejected': -104.581298828125, 'logits/chosen': -2.625985622406006, 'logits/rejected': -2.772733211517334, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.002680970588698983, 'epsilon_dpo/loss_margin_mean': 355.9402770996094, 'epsilon_dpo/beta_margin_mean': 0.951833963394165, 'epsilon_dpo/beta_margin_std': 0.6352096199989319, 'epsilon_dpo/beta_margin_grad_mean': -0.29450592398643494, 'epsilon_dpo/beta_margin_grad_std': 0.12352831661701202, 'kl/beta': 0.0027018103282898664, 'kl/avg_steps': 0.78125, 'epoch': 0.67}
|
|||
|
|
67%|████████████████████████████████████████████████████████████████████████████▋ | 458/681 [30:42<09:09, 2.46s/it]
67%|████████████████████████████████████████████████████████████████████████████▊ | 459/681 [30:45<09:07, 2.47s/it]
{'loss': 0.8004, 'grad_norm': 36.275413513183594, 'learning_rate': 1.466771464027316e-07, 'rewards/chosen': -0.4260733127593994, 'rewards/rejected': -1.2596666812896729, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.8335932493209839, 'logps/chosen': -192.38290405273438, 'logps/rejected': -565.0028076171875, 'logps/ref_chosen': -32.51487350463867, 'logps/ref_rejected': -90.99087524414062, 'logits/chosen': -2.501951217651367, 'logits/rejected': -2.667633533477783, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.002659349935129285, 'epsilon_dpo/loss_margin_mean': 314.1438903808594, 'epsilon_dpo/beta_margin_mean': 0.8335932493209839, 'epsilon_dpo/beta_margin_std': 0.6189974546432495, 'epsilon_dpo/beta_margin_grad_mean': -0.31732696294784546, 'epsilon_dpo/beta_margin_grad_std': 0.12530642747879028, 'kl/beta': 0.0026808660477399826, 'kl/avg_steps': 0.8125, 'epoch': 0.67}
|
|||
|
|
67%|████████████████████████████████████████████████████████████████████████████▊ | 459/681 [30:45<09:07, 2.47s/it]
68%|█████████████████████████████████████████████████████████████████████████████ | 460/681 [30:47<09:10, 2.49s/it]
{'loss': 0.7211, 'grad_norm': 46.03992462158203, 'learning_rate': 1.4550991377830423e-07, 'rewards/chosen': -0.4630896747112274, 'rewards/rejected': -1.4168858528137207, 'rewards/accuracies': 0.96875, 'rewards/margins': 0.9537962079048157, 'logps/chosen': -206.57542419433594, 'logps/rejected': -648.7615966796875, 'logps/ref_chosen': -31.02279281616211, 'logps/ref_rejected': -110.9461669921875, 'logits/chosen': -2.543520212173462, 'logits/rejected': -2.7920000553131104, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.0026354235596954823, 'epsilon_dpo/loss_margin_mean': 362.2628479003906, 'epsilon_dpo/beta_margin_mean': 0.9537962079048157, 'epsilon_dpo/beta_margin_std': 0.6000516414642334, 'epsilon_dpo/beta_margin_grad_mean': -0.2933025658130646, 'epsilon_dpo/beta_margin_grad_std': 0.11362046003341675, 'kl/beta': 0.0026592593640089035, 'kl/avg_steps': 0.90625, 'epoch': 0.68}
|
|||
|
|
68%|█████████████████████████████████████████████████████████████████████████████ | 460/681 [30:47<09:10, 2.49s/it]
68%|█████████████████████████████████████████████████████████████████████████████▏ | 461/681 [30:50<09:04, 2.48s/it]
{'loss': 0.7926, 'grad_norm': 37.75797653198242, 'learning_rate': 1.4434543456482518e-07, 'rewards/chosen': -0.44818025827407837, 'rewards/rejected': -1.2731022834777832, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.8249219655990601, 'logps/chosen': -206.56666564941406, 'logps/rejected': -580.9017333984375, 'logps/ref_chosen': -35.32524108886719, 'logps/ref_rejected': -93.41868591308594, 'logits/chosen': -2.542848587036133, 'logits/rejected': -2.7294955253601074, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.002612577984109521, 'epsilon_dpo/loss_margin_mean': 316.2416687011719, 'epsilon_dpo/beta_margin_mean': 0.8249220252037048, 'epsilon_dpo/beta_margin_std': 0.5639069676399231, 'epsilon_dpo/beta_margin_grad_mean': -0.31691277027130127, 'epsilon_dpo/beta_margin_grad_std': 0.11491651087999344, 'kl/beta': 0.0026353762950748205, 'kl/avg_steps': 0.875, 'epoch': 0.68}
|
|||
|
|
68%|█████████████████████████████████████████████████████████████████████████████▏ | 461/681 [30:50<09:04, 2.48s/it]
68%|█████████████████████████████████████████████████████████████████████████████▎ | 462/681 [30:52<08:56, 2.45s/it]
{'loss': 0.8226, 'grad_norm': 40.553749084472656, 'learning_rate': 1.4318373944740484e-07, 'rewards/chosen': -0.446227490901947, 'rewards/rejected': -1.2592161893844604, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.8129886984825134, 'logps/chosen': -216.20462036132812, 'logps/rejected': -571.200439453125, 'logps/ref_chosen': -44.890872955322266, 'logps/ref_rejected': -85.42142486572266, 'logits/chosen': -2.639768123626709, 'logits/rejected': -2.776127338409424, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0025939985644072294, 'epsilon_dpo/loss_margin_mean': 314.4653015136719, 'epsilon_dpo/beta_margin_mean': 0.8129886984825134, 'epsilon_dpo/beta_margin_std': 0.6604252457618713, 'epsilon_dpo/beta_margin_grad_mean': -0.3237418830394745, 'epsilon_dpo/beta_margin_grad_std': 0.13112208247184753, 'kl/beta': 0.00261251674965024, 'kl/avg_steps': 0.71875, 'epoch': 0.68}
|
|||
|
|
68%|█████████████████████████████████████████████████████████████████████████████▎ | 462/681 [30:52<08:56, 2.45s/it]
68%|█████████████████████████████████████████████████████████████████████████████▌ | 463/681 [30:55<09:11, 2.53s/it]
{'loss': 0.712, 'grad_norm': 36.294189453125, 'learning_rate': 1.4202485903778976e-07, 'rewards/chosen': -0.32232174277305603, 'rewards/rejected': -1.3272720575332642, 'rewards/accuracies': 0.953125, 'rewards/margins': 1.0049502849578857, 'logps/chosen': -152.20263671875, 'logps/rejected': -614.646240234375, 'logps/ref_chosen': -27.038963317871094, 'logps/ref_rejected': -98.03726196289062, 'logits/chosen': -2.6299610137939453, 'logits/rejected': -2.773591995239258, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.0025706232991069555, 'epsilon_dpo/loss_margin_mean': 391.4453125, 'epsilon_dpo/beta_margin_mean': 1.0049502849578857, 'epsilon_dpo/beta_margin_std': 0.6909827589988708, 'epsilon_dpo/beta_margin_grad_mean': -0.28741469979286194, 'epsilon_dpo/beta_margin_grad_std': 0.12530295550823212, 'kl/beta': 0.0025938733015209436, 'kl/avg_steps': 0.90625, 'epoch': 0.68}
|
|||
|
|
68%|█████████████████████████████████████████████████████████████████████████████▌ | 463/681 [30:55<09:11, 2.53s/it]
68%|█████████████████████████████████████████████████████████████████████████████▋ | 464/681 [30:57<09:20, 2.58s/it]
{'loss': 0.6506, 'grad_norm': 48.614654541015625, 'learning_rate': 1.4086882387355658e-07, 'rewards/chosen': -0.3993784487247467, 'rewards/rejected': -1.5100371837615967, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.1106586456298828, 'logps/chosen': -189.71588134765625, 'logps/rejected': -701.7393798828125, 'logps/ref_chosen': -33.55242919921875, 'logps/ref_rejected': -109.08905029296875, 'logits/chosen': -2.5940399169921875, 'logits/rejected': -2.805851697921753, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.00254994654096663, 'epsilon_dpo/loss_margin_mean': 436.4868469238281, 'epsilon_dpo/beta_margin_mean': 1.1106587648391724, 'epsilon_dpo/beta_margin_std': 0.6612549424171448, 'epsilon_dpo/beta_margin_grad_mean': -0.2655731737613678, 'epsilon_dpo/beta_margin_grad_std': 0.12443613260984421, 'kl/beta': 0.0025705774314701557, 'kl/avg_steps': 0.8125, 'epoch': 0.68}
|
|||
|
|
68%|█████████████████████████████████████████████████████████████████████████████▋ | 464/681 [30:57<09:20, 2.58s/it]
68%|█████████████████████████████████████████████████████████████████████████████▊ | 465/681 [31:00<09:09, 2.54s/it]
{'loss': 0.7035, 'grad_norm': 35.46124267578125, 'learning_rate': 1.3971566441730714e-07, 'rewards/chosen': -0.36298391222953796, 'rewards/rejected': -1.3539800643920898, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.9909961223602295, 'logps/chosen': -178.53143310546875, 'logps/rejected': -652.1649780273438, 'logps/ref_chosen': -35.28005599975586, 'logps/ref_rejected': -116.3499755859375, 'logits/chosen': -2.6725172996520996, 'logits/rejected': -2.8897171020507812, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0025285982992500067, 'epsilon_dpo/loss_margin_mean': 392.5635986328125, 'epsilon_dpo/beta_margin_mean': 0.9909960627555847, 'epsilon_dpo/beta_margin_std': 0.621567964553833, 'epsilon_dpo/beta_margin_grad_mean': -0.28670158982276917, 'epsilon_dpo/beta_margin_grad_std': 0.11507753282785416, 'kl/beta': 0.0025498599279671907, 'kl/avg_steps': 0.84375, 'epoch': 0.68}
|
|||
|
|
68%|█████████████████████████████████████████████████████████████████████████████▊ | 465/681 [31:00<09:09, 2.54s/it]
68%|██████████████████████████████████████████████████████████████████████████████ | 466/681 [31:02<09:08, 2.55s/it]
{'loss': 0.749, 'grad_norm': 32.29966354370117, 'learning_rate': 1.3856541105586545e-07, 'rewards/chosen': -0.3691789507865906, 'rewards/rejected': -1.2763590812683105, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.9071800112724304, 'logps/chosen': -175.187255859375, 'logps/rejected': -607.10595703125, 'logps/ref_chosen': -28.18646240234375, 'logps/ref_rejected': -97.64432525634766, 'logits/chosen': -2.5877268314361572, 'logits/rejected': -2.7581279277801514, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0025074416771531105, 'epsilon_dpo/loss_margin_mean': 362.46087646484375, 'epsilon_dpo/beta_margin_mean': 0.9071800112724304, 'epsilon_dpo/beta_margin_std': 0.6066918969154358, 'epsilon_dpo/beta_margin_grad_mean': -0.3023064434528351, 'epsilon_dpo/beta_margin_grad_std': 0.11443696171045303, 'kl/beta': 0.002528525423258543, 'kl/avg_steps': 0.84375, 'epoch': 0.68}
|
|||
|
|
68%|██████████████████████████████████████████████████████████████████████████████ | 466/681 [31:02<09:08, 2.55s/it]
69%|██████████████████████████████████████████████████████████████████████████████▏ | 467/681 [31:05<09:00, 2.52s/it]
{'loss': 0.7652, 'grad_norm': 47.59917449951172, 'learning_rate': 1.3741809409947729e-07, 'rewards/chosen': -0.44280189275741577, 'rewards/rejected': -1.4351122379302979, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.9923102259635925, 'logps/chosen': -223.679443359375, 'logps/rejected': -687.1947021484375, 'logps/ref_chosen': -46.7025146484375, 'logps/ref_rejected': -110.00337982177734, 'logits/chosen': -2.691765308380127, 'logits/rejected': -2.9094629287719727, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0024880296550691128, 'epsilon_dpo/loss_margin_mean': 400.21441650390625, 'epsilon_dpo/beta_margin_mean': 0.9923102259635925, 'epsilon_dpo/beta_margin_std': 0.829995334148407, 'epsilon_dpo/beta_margin_grad_mean': -0.2942372262477875, 'epsilon_dpo/beta_margin_grad_std': 0.15898768603801727, 'kl/beta': 0.002507369499653578, 'kl/avg_steps': 0.78125, 'epoch': 0.69}
|
|||
|
|
69%|██████████████████████████████████████████████████████████████████████████████▏ | 467/681 [31:05<09:00, 2.52s/it]
69%|██████████████████████████████████████████████████████████████████████████████▎ | 468/681 [31:07<09:05, 2.56s/it]
{'loss': 0.728, 'grad_norm': 40.286834716796875, 'learning_rate': 1.362737437810114e-07, 'rewards/chosen': -0.37121495604515076, 'rewards/rejected': -1.3838683366775513, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.0126533508300781, 'logps/chosen': -192.12745666503906, 'logps/rejected': -670.7412109375, 'logps/ref_chosen': -42.05735778808594, 'logps/ref_rejected': -109.48826599121094, 'logits/chosen': -2.67857027053833, 'logits/rejected': -2.87001895904541, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0024679647758603096, 'epsilon_dpo/loss_margin_mean': 411.1828918457031, 'epsilon_dpo/beta_margin_mean': 1.0126533508300781, 'epsilon_dpo/beta_margin_std': 0.7630032896995544, 'epsilon_dpo/beta_margin_grad_mean': -0.29025694727897644, 'epsilon_dpo/beta_margin_grad_std': 0.14004936814308167, 'kl/beta': 0.002487932564690709, 'kl/avg_steps': 0.8125, 'epoch': 0.69}
|
|||
|
|
69%|██████████████████████████████████████████████████████████████████████████████▎ | 468/681 [31:08<09:05, 2.56s/it]
69%|██████████████████████████████████████████████████████████████████████████████▌ | 469/681 [31:10<08:47, 2.49s/it]
{'loss': 0.7019, 'grad_norm': 48.018531799316406, 'learning_rate': 1.351323902551631e-07, 'rewards/chosen': -0.4174906611442566, 'rewards/rejected': -1.417689561843872, 'rewards/accuracies': 0.96875, 'rewards/margins': 1.0001988410949707, 'logps/chosen': -206.51370239257812, 'logps/rejected': -690.9351806640625, 'logps/ref_chosen': -36.20582580566406, 'logps/ref_rejected': -111.1355972290039, 'logits/chosen': -2.6168570518493652, 'logits/rejected': -2.8470990657806396, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.002445760415866971, 'epsilon_dpo/loss_margin_mean': 409.49169921875, 'epsilon_dpo/beta_margin_mean': 1.0001988410949707, 'epsilon_dpo/beta_margin_std': 0.6260125637054443, 'epsilon_dpo/beta_margin_grad_mean': -0.2844110131263733, 'epsilon_dpo/beta_margin_grad_std': 0.1195027157664299, 'kl/beta': 0.002467880956828594, 'kl/avg_steps': 0.90625, 'epoch': 0.69}
|
|||
|
|
69%|██████████████████████████████████████████████████████████████████████████████▌ | 469/681 [31:10<08:47, 2.49s/it]
69%|██████████████████████████████████████████████████████████████████████████████▋ | 470/681 [31:12<08:30, 2.42s/it]
{'loss': 0.7329, 'grad_norm': 35.75682830810547, 'learning_rate': 1.339940635976592e-07, 'rewards/chosen': -0.30687910318374634, 'rewards/rejected': -1.2565128803253174, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.9496338367462158, 'logps/chosen': -157.40719604492188, 'logps/rejected': -605.90576171875, 'logps/ref_chosen': -31.09160804748535, 'logps/ref_rejected': -87.30916595458984, 'logits/chosen': -2.6013760566711426, 'logits/rejected': -2.87990665435791, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.002425323473289609, 'epsilon_dpo/loss_margin_mean': 392.281005859375, 'epsilon_dpo/beta_margin_mean': 0.9496338963508606, 'epsilon_dpo/beta_margin_std': 0.6434259414672852, 'epsilon_dpo/beta_margin_grad_mean': -0.2954522371292114, 'epsilon_dpo/beta_margin_grad_std': 0.12080518156290054, 'kl/beta': 0.002445716643705964, 'kl/avg_steps': 0.84375, 'epoch': 0.69}
|
|||
|
|
69%|██████████████████████████████████████████████████████████████████████████████▋ | 470/681 [31:12<08:30, 2.42s/it]
69%|██████████████████████████████████████████████████████████████████████████████▊ | 471/681 [31:14<08:19, 2.38s/it]
{'loss': 0.745, 'grad_norm': 35.150997161865234, 'learning_rate': 1.3285879380446563e-07, 'rewards/chosen': -0.4026249945163727, 'rewards/rejected': -1.3193340301513672, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.9167090654373169, 'logps/chosen': -211.00067138671875, 'logps/rejected': -638.4447631835938, 'logps/ref_chosen': -44.132484436035156, 'logps/ref_rejected': -89.70744323730469, 'logits/chosen': -2.7026596069335938, 'logits/rejected': -2.8277225494384766, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.002405789215117693, 'epsilon_dpo/loss_margin_mean': 381.869140625, 'epsilon_dpo/beta_margin_mean': 0.9167090654373169, 'epsilon_dpo/beta_margin_std': 0.606320858001709, 'epsilon_dpo/beta_margin_grad_mean': -0.30035099387168884, 'epsilon_dpo/beta_margin_grad_std': 0.11763791739940643, 'kl/beta': 0.002425253624096513, 'kl/avg_steps': 0.8125, 'epoch': 0.69}
|
|||
|
|
69%|██████████████████████████████████████████████████████████████████████████████▊ | 471/681 [31:14<08:19, 2.38s/it]
69%|███████████████████████████████████████████████████████████████████████████████ | 472/681 [31:17<08:34, 2.46s/it]
{'loss': 0.778, 'grad_norm': 36.8669548034668, 'learning_rate': 1.317266107909975e-07, 'rewards/chosen': -0.4928061366081238, 'rewards/rejected': -1.4490196704864502, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.9562135934829712, 'logps/chosen': -265.7175598144531, 'logps/rejected': -731.2271728515625, 'logps/ref_chosen': -59.834197998046875, 'logps/ref_rejected': -123.57960510253906, 'logits/chosen': -2.770143985748291, 'logits/rejected': -2.981727361679077, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.002386399544775486, 'epsilon_dpo/loss_margin_mean': 401.76422119140625, 'epsilon_dpo/beta_margin_mean': 0.9562135934829712, 'epsilon_dpo/beta_margin_std': 0.8416454792022705, 'epsilon_dpo/beta_margin_grad_mean': -0.3045946955680847, 'epsilon_dpo/beta_margin_grad_std': 0.14598874747753143, 'kl/beta': 0.002405707258731127, 'kl/avg_steps': 0.8125, 'epoch': 0.69}
|
|||
|
|
69%|███████████████████████████████████████████████████████████████████████████████ | 472/681 [31:17<08:34, 2.46s/it]
69%|███████████████████████████████████████████████████████████████████████████████▏ | 473/681 [31:20<08:42, 2.51s/it]
{'loss': 0.8578, 'grad_norm': 50.768165588378906, 'learning_rate': 1.3059754439133002e-07, 'rewards/chosen': -0.4595049023628235, 'rewards/rejected': -1.2219587564468384, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.7624538540840149, 'logps/chosen': -238.464599609375, 'logps/rejected': -602.4376220703125, 'logps/ref_chosen': -44.87860870361328, 'logps/ref_rejected': -85.82889556884766, 'logits/chosen': -2.749878406524658, 'logits/rejected': -2.8519415855407715, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0023671663366258144, 'epsilon_dpo/loss_margin_mean': 323.02276611328125, 'epsilon_dpo/beta_margin_mean': 0.7624538540840149, 'epsilon_dpo/beta_margin_std': 0.6814138889312744, 'epsilon_dpo/beta_margin_grad_mean': -0.3334435224533081, 'epsilon_dpo/beta_margin_grad_std': 0.130352184176445, 'kl/beta': 0.0023863185197114944, 'kl/avg_steps': 0.8125, 'epoch': 0.69}
|
|||
|
|
69%|███████████████████████████████████████████████████████████████████████████████▏ | 473/681 [31:20<08:42, 2.51s/it]
70%|███████████████████████████████████████████████████████████████████████████████▎ | 474/681 [31:22<08:45, 2.54s/it]
{'loss': 0.7665, 'grad_norm': 44.010650634765625, 'learning_rate': 1.2947162435741277e-07, 'rewards/chosen': -0.38724249601364136, 'rewards/rejected': -1.2767717838287354, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.889529287815094, 'logps/chosen': -195.01571655273438, 'logps/rejected': -641.80322265625, 'logps/ref_chosen': -30.269367218017578, 'logps/ref_rejected': -97.37470245361328, 'logits/chosen': -2.6113648414611816, 'logits/rejected': -2.8044967651367188, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0023473482578992844, 'epsilon_dpo/loss_margin_mean': 379.68218994140625, 'epsilon_dpo/beta_margin_mean': 0.889529287815094, 'epsilon_dpo/beta_margin_std': 0.6321231126785278, 'epsilon_dpo/beta_margin_grad_mean': -0.3070417642593384, 'epsilon_dpo/beta_margin_grad_std': 0.12120737135410309, 'kl/beta': 0.00236708577722311, 'kl/avg_steps': 0.84375, 'epoch': 0.7}
|
|||
|
|
70%|███████████████████████████████████████████████████████████████████████████████▎ | 474/681 [31:22<08:45, 2.54s/it]
70%|███████████████████████████████████████████████████████████████████████████████▌ | 475/681 [31:25<08:43, 2.54s/it]
{'loss': 0.8274, 'grad_norm': 35.23362350463867, 'learning_rate': 1.2834888035828596e-07, 'rewards/chosen': -0.4007193148136139, 'rewards/rejected': -1.2023236751556396, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.8016044497489929, 'logps/chosen': -206.45596313476562, 'logps/rejected': -611.5294189453125, 'logps/ref_chosen': -34.96168518066406, 'logps/ref_rejected': -94.91036987304688, 'logits/chosen': -2.689065933227539, 'logits/rejected': -2.8997483253479004, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.002328441943973303, 'epsilon_dpo/loss_margin_mean': 345.1247863769531, 'epsilon_dpo/beta_margin_mean': 0.8016044497489929, 'epsilon_dpo/beta_margin_std': 0.6676663160324097, 'epsilon_dpo/beta_margin_grad_mean': -0.32659581303596497, 'epsilon_dpo/beta_margin_grad_std': 0.1249208152294159, 'kl/beta': 0.0023472807370126247, 'kl/avg_steps': 0.8125, 'epoch': 0.7}
|
|||
|
|
70%|███████████████████████████████████████████████████████████████████████████████▌ | 475/681 [31:25<08:43, 2.54s/it]
70%|███████████████████████████████████████████████████████████████████████████████▋ | 476/681 [31:27<08:24, 2.46s/it]
{'loss': 0.8208, 'grad_norm': 40.693206787109375, 'learning_rate': 1.2722934197929802e-07, 'rewards/chosen': -0.37969422340393066, 'rewards/rejected': -1.183998942375183, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.8043047189712524, 'logps/chosen': -192.21990966796875, 'logps/rejected': -591.2510986328125, 'logps/ref_chosen': -28.34685516357422, 'logps/ref_rejected': -78.29444885253906, 'logits/chosen': -2.668698310852051, 'logits/rejected': -2.884255886077881, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.002310403622686863, 'epsilon_dpo/loss_margin_mean': 349.0835876464844, 'epsilon_dpo/beta_margin_mean': 0.8043047189712524, 'epsilon_dpo/beta_margin_std': 0.6316364407539368, 'epsilon_dpo/beta_margin_grad_mean': -0.32414335012435913, 'epsilon_dpo/beta_margin_grad_std': 0.12600372731685638, 'kl/beta': 0.0023283627815544605, 'kl/avg_steps': 0.78125, 'epoch': 0.7}
|
|||
|
|
70%|███████████████████████████████████████████████████████████████████████████████▋ | 476/681 [31:27<08:24, 2.46s/it]
70%|███████████████████████████████████████████████████████████████████████████████▊ | 477/681 [31:30<08:28, 2.49s/it]
{'loss': 0.802, 'grad_norm': 37.30575942993164, 'learning_rate': 1.2611303872132631e-07, 'rewards/chosen': -0.40344566106796265, 'rewards/rejected': -1.219561219215393, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.8161154985427856, 'logps/chosen': -217.6685028076172, 'logps/rejected': -614.880126953125, 'logps/ref_chosen': -42.40911865234375, 'logps/ref_rejected': -82.68942260742188, 'logits/chosen': -2.7471470832824707, 'logits/rejected': -2.842437267303467, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0022924933582544327, 'epsilon_dpo/loss_margin_mean': 356.9313049316406, 'epsilon_dpo/beta_margin_mean': 0.8161155581474304, 'epsilon_dpo/beta_margin_std': 0.5828248858451843, 'epsilon_dpo/beta_margin_grad_mean': -0.3193974196910858, 'epsilon_dpo/beta_margin_grad_std': 0.11756820976734161, 'kl/beta': 0.002310313517227769, 'kl/avg_steps': 0.78125, 'epoch': 0.7}
|
|||
|
|
70%|███████████████████████████████████████████████████████████████████████████████▊ | 477/681 [31:30<08:28, 2.49s/it]
70%|████████████████████████████████████████████████████████████████████████████████ | 478/681 [31:32<08:10, 2.42s/it]
{'loss': 0.7954, 'grad_norm': 39.92375564575195, 'learning_rate': 1.2500000000000005e-07, 'rewards/chosen': -0.407898485660553, 'rewards/rejected': -1.2313954830169678, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.8234970569610596, 'logps/chosen': -207.5630340576172, 'logps/rejected': -632.2252197265625, 'logps/ref_chosen': -28.737815856933594, 'logps/ref_rejected': -90.47331237792969, 'logits/chosen': -2.6334609985351562, 'logits/rejected': -2.921363592147827, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0022747220937162638, 'epsilon_dpo/loss_margin_mean': 362.92669677734375, 'epsilon_dpo/beta_margin_mean': 0.8234969973564148, 'epsilon_dpo/beta_margin_std': 0.577464759349823, 'epsilon_dpo/beta_margin_grad_mean': -0.3177875578403473, 'epsilon_dpo/beta_margin_grad_std': 0.11528382450342178, 'kl/beta': 0.0022924039512872696, 'kl/avg_steps': 0.78125, 'epoch': 0.7}
|
|||
|
|
70%|████████████████████████████████████████████████████████████████████████████████ | 478/681 [31:32<08:10, 2.42s/it]
70%|████████████████████████████████████████████████████████████████████████████████▏ | 479/681 [31:34<08:13, 2.44s/it]
{'loss': 0.7896, 'grad_norm': 42.03719711303711, 'learning_rate': 1.2389025514492456e-07, 'rewards/chosen': -0.37124890089035034, 'rewards/rejected': -1.2771828174591064, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.9059340357780457, 'logps/chosen': -194.02374267578125, 'logps/rejected': -670.686767578125, 'logps/ref_chosen': -29.93898582458496, 'logps/ref_rejected': -104.23573303222656, 'logits/chosen': -2.710920810699463, 'logits/rejected': -3.0226893424987793, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.002257088664919138, 'epsilon_dpo/loss_margin_mean': 402.3663330078125, 'epsilon_dpo/beta_margin_mean': 0.9059340357780457, 'epsilon_dpo/beta_margin_std': 0.7668508291244507, 'epsilon_dpo/beta_margin_grad_mean': -0.3112712800502777, 'epsilon_dpo/beta_margin_grad_std': 0.1403777301311493, 'kl/beta': 0.0022746333852410316, 'kl/avg_steps': 0.78125, 'epoch': 0.7}
|
|||
|
|
70%|████████████████████████████████████████████████████████████████████████████████▏ | 479/681 [31:34<08:13, 2.44s/it]
70%|████████████████████████████████████████████████████████████████████████████████▎ | 480/681 [31:37<08:06, 2.42s/it]
{'loss': 0.8298, 'grad_norm': 41.93212127685547, 'learning_rate': 1.227838333989088e-07, 'rewards/chosen': -0.4799705743789673, 'rewards/rejected': -1.2608397006988525, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.7808691263198853, 'logps/chosen': -257.56707763671875, 'logps/rejected': -650.7171020507812, 'logps/ref_chosen': -43.97426223754883, 'logps/ref_rejected': -87.41323852539062, 'logits/chosen': -2.7833218574523926, 'logits/rejected': -2.883632183074951, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0022395916748791933, 'epsilon_dpo/loss_margin_mean': 349.7110595703125, 'epsilon_dpo/beta_margin_mean': 0.7808691263198853, 'epsilon_dpo/beta_margin_std': 0.6041759848594666, 'epsilon_dpo/beta_margin_grad_mean': -0.3270338773727417, 'epsilon_dpo/beta_margin_grad_std': 0.12246443331241608, 'kl/beta': 0.002257000654935837, 'kl/avg_steps': 0.78125, 'epoch': 0.7}
|
|||
|
|
70%|████████████████████████████████████████████████████████████████████████████████▎ | 480/681 [31:37<08:06, 2.42s/it]
71%|████████████████████████████████████████████████████████████████████████████████▌ | 481/681 [31:39<08:10, 2.45s/it]
{'loss': 0.8128, 'grad_norm': 38.608699798583984, 'learning_rate': 1.2168076391719489e-07, 'rewards/chosen': -0.4300834834575653, 'rewards/rejected': -1.267479658126831, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.8373961448669434, 'logps/chosen': -229.94204711914062, 'logps/rejected': -669.5849609375, 'logps/ref_chosen': -36.98882293701172, 'logps/ref_rejected': -98.65377807617188, 'logits/chosen': -2.710186004638672, 'logits/rejected': -3.0134506225585938, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0022222306579351425, 'epsilon_dpo/loss_margin_mean': 377.9779968261719, 'epsilon_dpo/beta_margin_mean': 0.8373962044715881, 'epsilon_dpo/beta_margin_std': 0.6917349100112915, 'epsilon_dpo/beta_margin_grad_mean': -0.31972822546958923, 'epsilon_dpo/beta_margin_grad_std': 0.13162846863269806, 'kl/beta': 0.0022395045962184668, 'kl/avg_steps': 0.78125, 'epoch': 0.71}
|
|||
|
|
71%|████████████████████████████████████████████████████████████████████████████████▌ | 481/681 [31:39<08:10, 2.45s/it]
71%|████████████████████████████████████████████████████████████████████████████████▋ | 482/681 [31:42<08:17, 2.50s/it]
{'loss': 0.875, 'grad_norm': 43.73499298095703, 'learning_rate': 1.2058107576668938e-07, 'rewards/chosen': -0.4530524015426636, 'rewards/rejected': -1.1681525707244873, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.7151001691818237, 'logps/chosen': -252.25537109375, 'logps/rejected': -622.5506591796875, 'logps/ref_chosen': -47.419219970703125, 'logps/ref_rejected': -92.47096252441406, 'logits/chosen': -2.777127504348755, 'logits/rejected': -3.0118846893310547, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0022050042171031237, 'epsilon_dpo/loss_margin_mean': 325.2435607910156, 'epsilon_dpo/beta_margin_mean': 0.7151001691818237, 'epsilon_dpo/beta_margin_std': 0.6155868172645569, 'epsilon_dpo/beta_margin_grad_mean': -0.34148189425468445, 'epsilon_dpo/beta_margin_grad_std': 0.12353808432817459, 'kl/beta': 0.0022221440449357033, 'kl/avg_steps': 0.78125, 'epoch': 0.71}
|
|||
|
|
71%|████████████████████████████████████████████████████████████████████████████████▋ | 482/681 [31:42<08:17, 2.50s/it]
71%|████████████████████████████████████████████████████████████████████████████████▊ | 483/681 [31:44<08:14, 2.50s/it]
{'loss': 0.7801, 'grad_norm': 37.85783386230469, 'learning_rate': 1.194847979251979e-07, 'rewards/chosen': -0.39537718892097473, 'rewards/rejected': -1.2726057767868042, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.8772286176681519, 'logps/chosen': -219.78411865234375, 'logps/rejected': -683.1080322265625, 'logps/ref_chosen': -39.672393798828125, 'logps/ref_rejected': -100.94681549072266, 'logits/chosen': -2.7550508975982666, 'logits/rejected': -3.005962371826172, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0021872217766940594, 'epsilon_dpo/loss_margin_mean': 402.0494689941406, 'epsilon_dpo/beta_margin_mean': 0.8772286176681519, 'epsilon_dpo/beta_margin_std': 0.6511039733886719, 'epsilon_dpo/beta_margin_grad_mean': -0.3099426329135895, 'epsilon_dpo/beta_margin_grad_std': 0.1277041733264923, 'kl/beta': 0.0022049180697649717, 'kl/avg_steps': 0.8125, 'epoch': 0.71}
|
|||
|
|
71%|████████████████████████████████████████████████████████████████████████████████▊ | 483/681 [31:44<08:14, 2.50s/it]
71%|█████████████████████████████████████████████████████████████████████████████████ | 484/681 [31:47<08:11, 2.50s/it]
{'loss': 0.9168, 'grad_norm': 40.76497268676758, 'learning_rate': 1.1839195928066101e-07, 'rewards/chosen': -0.4243118464946747, 'rewards/rejected': -1.095069169998169, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.6707573533058167, 'logps/chosen': -237.5784149169922, 'logps/rejected': -594.3012084960938, 'logps/ref_chosen': -43.43277359008789, 'logps/ref_rejected': -89.96736145019531, 'logits/chosen': -2.699538230895996, 'logits/rejected': -2.9171078205108643, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.002174378838390112, 'epsilon_dpo/loss_margin_mean': 310.1882019042969, 'epsilon_dpo/beta_margin_mean': 0.6707573533058167, 'epsilon_dpo/beta_margin_std': 0.6563224792480469, 'epsilon_dpo/beta_margin_grad_mean': -0.35192134976387024, 'epsilon_dpo/beta_margin_grad_std': 0.1350928395986557, 'kl/beta': 0.002187147503718734, 'kl/avg_steps': 0.59375, 'epoch': 0.71}
|
|||
|
|
71%|█████████████████████████████████████████████████████████████████████████████████ | 484/681 [31:47<08:11, 2.50s/it]
71%|█████████████████████████████████████████████████████████████████████████████████▏ | 485/681 [31:49<07:49, 2.40s/it]
{'loss': 0.7557, 'grad_norm': 32.71714782714844, 'learning_rate': 1.1730258863039347e-07, 'rewards/chosen': -0.28553080558776855, 'rewards/rejected': -1.2164714336395264, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.9309406876564026, 'logps/chosen': -169.16915893554688, 'logps/rejected': -675.8591918945312, 'logps/ref_chosen': -37.34454345703125, 'logps/ref_rejected': -111.42447662353516, 'logits/chosen': -2.6770548820495605, 'logits/rejected': -2.999387264251709, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0021574674174189568, 'epsilon_dpo/loss_margin_mean': 432.610107421875, 'epsilon_dpo/beta_margin_mean': 0.9309406876564026, 'epsilon_dpo/beta_margin_std': 0.6903952360153198, 'epsilon_dpo/beta_margin_grad_mean': -0.3015494644641876, 'epsilon_dpo/beta_margin_grad_std': 0.12973572313785553, 'kl/beta': 0.0021742379758507013, 'kl/avg_steps': 0.78125, 'epoch': 0.71}
|
|||
|
|
71%|█████████████████████████████████████████████████████████████████████████████████▏ | 485/681 [31:49<07:49, 2.40s/it]
71%|█████████████████████████████████████████████████████████████████████████████████▎ | 486/681 [31:51<07:35, 2.34s/it]
{'loss': 0.8285, 'grad_norm': 35.33386993408203, 'learning_rate': 1.1621671468032493e-07, 'rewards/chosen': -0.34016796946525574, 'rewards/rejected': -1.1584980487823486, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.8183300495147705, 'logps/chosen': -193.83453369140625, 'logps/rejected': -640.993896484375, 'logps/ref_chosen': -35.52677536010742, 'logps/ref_rejected': -99.17495727539062, 'logits/chosen': -2.755924701690674, 'logits/rejected': -3.058840274810791, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.002140068681910634, 'epsilon_dpo/loss_margin_mean': 383.51116943359375, 'epsilon_dpo/beta_margin_mean': 0.8183300495147705, 'epsilon_dpo/beta_margin_std': 0.6934942007064819, 'epsilon_dpo/beta_margin_grad_mean': -0.3228636085987091, 'epsilon_dpo/beta_margin_grad_std': 0.1375182420015335, 'kl/beta': 0.0021573833655565977, 'kl/avg_steps': 0.8125, 'epoch': 0.71}
|
|||
|
|
71%|█████████████████████████████████████████████████████████████████████████████████▎ | 486/681 [31:51<07:35, 2.34s/it]
72%|█████████████████████████████████████████████████████████████████████████████████▌ | 487/681 [31:54<07:55, 2.45s/it]
{'loss': 0.763, 'grad_norm': 43.43318557739258, 'learning_rate': 1.1513436604424378e-07, 'rewards/chosen': -0.3262515366077423, 'rewards/rejected': -1.1740999221801758, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.8478483557701111, 'logps/chosen': -184.3863525390625, 'logps/rejected': -652.44091796875, 'logps/ref_chosen': -31.08715057373047, 'logps/ref_rejected': -98.84352111816406, 'logits/chosen': -2.668581247329712, 'logits/rejected': -3.001206636428833, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0021214832086116076, 'epsilon_dpo/loss_margin_mean': 400.2981872558594, 'epsilon_dpo/beta_margin_mean': 0.8478483557701111, 'epsilon_dpo/beta_margin_std': 0.48607146739959717, 'epsilon_dpo/beta_margin_grad_mean': -0.30856984853744507, 'epsilon_dpo/beta_margin_grad_std': 0.10273387283086777, 'kl/beta': 0.002139996038749814, 'kl/avg_steps': 0.875, 'epoch': 0.72}
|
|||
|
|
72%|█████████████████████████████████████████████████████████████████████████████████▌ | 487/681 [31:54<07:55, 2.45s/it]
72%|█████████████████████████████████████████████████████████████████████████████████▋ | 488/681 [31:57<08:03, 2.51s/it]
{'loss': 0.8576, 'grad_norm': 35.68146514892578, 'learning_rate': 1.1405557124304335e-07, 'rewards/chosen': -0.3142842650413513, 'rewards/rejected': -1.047201156616211, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.7329168319702148, 'logps/chosen': -184.39353942871094, 'logps/rejected': -587.299072265625, 'logps/ref_chosen': -35.27953338623047, 'logps/ref_rejected': -89.09225463867188, 'logits/chosen': -2.7139856815338135, 'logits/rejected': -2.9660186767578125, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0021050700452178717, 'epsilon_dpo/loss_margin_mean': 349.0928039550781, 'epsilon_dpo/beta_margin_mean': 0.7329168319702148, 'epsilon_dpo/beta_margin_std': 0.5872458815574646, 'epsilon_dpo/beta_margin_grad_mean': -0.33707907795906067, 'epsilon_dpo/beta_margin_grad_std': 0.12150773406028748, 'kl/beta': 0.0021214333828538656, 'kl/avg_steps': 0.78125, 'epoch': 0.72}
|
|||
|
|
72%|█████████████████████████████████████████████████████████████████████████████████▋ | 488/681 [31:57<08:03, 2.51s/it]
72%|█████████████████████████████████████████████████████████████████████████████████▊ | 489/681 [31:59<08:14, 2.58s/it]
{'loss': 0.8644, 'grad_norm': 40.08867263793945, 'learning_rate': 1.1298035870396985e-07, 'rewards/chosen': -0.3389902710914612, 'rewards/rejected': -1.0370516777038574, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.6980613470077515, 'logps/chosen': -199.35702514648438, 'logps/rejected': -584.06689453125, 'logps/ref_chosen': -37.423851013183594, 'logps/ref_rejected': -87.10142517089844, 'logits/chosen': -2.7437334060668945, 'logits/rejected': -2.992414712905884, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0020880938973277807, 'epsilon_dpo/loss_margin_mean': 335.0322570800781, 'epsilon_dpo/beta_margin_mean': 0.6980612874031067, 'epsilon_dpo/beta_margin_std': 0.5159537196159363, 'epsilon_dpo/beta_margin_grad_mean': -0.34188124537467957, 'epsilon_dpo/beta_margin_grad_std': 0.10775003582239151, 'kl/beta': 0.0021049880888313055, 'kl/avg_steps': 0.8125, 'epoch': 0.72}
|
|||
|
|
72%|█████████████████████████████████████████████████████████████████████████████████▊ | 489/681 [31:59<08:14, 2.58s/it]
72%|██████████████████████████████████████████████████████████████████████████████████ | 490/681 [32:02<08:16, 2.60s/it]
{'loss': 0.9131, 'grad_norm': 42.104305267333984, 'learning_rate': 1.1190875675987355e-07, 'rewards/chosen': -0.42905905842781067, 'rewards/rejected': -1.1452784538269043, 'rewards/accuracies': 0.796875, 'rewards/margins': 0.7162193059921265, 'logps/chosen': -247.21926879882812, 'logps/rejected': -668.4090576171875, 'logps/ref_chosen': -41.46424102783203, 'logps/ref_rejected': -115.67326354980469, 'logits/chosen': -2.7477211952209473, 'logits/rejected': -3.100935935974121, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.002076485427096486, 'epsilon_dpo/loss_margin_mean': 346.98077392578125, 'epsilon_dpo/beta_margin_mean': 0.7162193655967712, 'epsilon_dpo/beta_margin_std': 0.7605259418487549, 'epsilon_dpo/beta_margin_grad_mean': -0.3476724922657013, 'epsilon_dpo/beta_margin_grad_std': 0.1495998054742813, 'kl/beta': 0.0020880228839814663, 'kl/avg_steps': 0.5625, 'epoch': 0.72}
|
|||
|
|
72%|██████████████████████████████████████████████████████████████████████████████████ | 490/681 [32:02<08:16, 2.60s/it]
72%|██████████████████████████████████████████████████████████████████████████████████▏ | 491/681 [32:04<08:08, 2.57s/it]
{'loss': 0.9096, 'grad_norm': 40.64850997924805, 'learning_rate': 1.1084079364846241e-07, 'rewards/chosen': -0.38356825709342957, 'rewards/rejected': -1.0272889137268066, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.6437206268310547, 'logps/chosen': -226.95785522460938, 'logps/rejected': -578.698974609375, 'logps/ref_chosen': -41.33907699584961, 'logps/ref_rejected': -79.69932556152344, 'logits/chosen': -2.7214245796203613, 'logits/rejected': -3.0119452476501465, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.002060000551864505, 'epsilon_dpo/loss_margin_mean': 313.3808288574219, 'epsilon_dpo/beta_margin_mean': 0.6437206268310547, 'epsilon_dpo/beta_margin_std': 0.547960102558136, 'epsilon_dpo/beta_margin_grad_mean': -0.3543255031108856, 'epsilon_dpo/beta_margin_grad_std': 0.11612068116664886, 'kl/beta': 0.002076343633234501, 'kl/avg_steps': 0.796875, 'epoch': 0.72}
|
|||
|
|
72%|██████████████████████████████████████████████████████████████████████████████████▏ | 491/681 [32:04<08:08, 2.57s/it]
72%|██████████████████████████████████████████████████████████████████████████████████▎ | 492/681 [32:07<07:47, 2.47s/it]
{'loss': 0.929, 'grad_norm': 34.33412551879883, 'learning_rate': 1.097764975115576e-07, 'rewards/chosen': -0.3345920443534851, 'rewards/rejected': -0.9495918154716492, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.6149997711181641, 'logps/chosen': -195.3032989501953, 'logps/rejected': -544.8341064453125, 'logps/ref_chosen': -31.90703582763672, 'logps/ref_rejected': -79.67924499511719, 'logits/chosen': -2.7548234462738037, 'logits/rejected': -2.9680824279785156, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0020440397784113884, 'epsilon_dpo/loss_margin_mean': 301.7585754394531, 'epsilon_dpo/beta_margin_mean': 0.6149997711181641, 'epsilon_dpo/beta_margin_std': 0.5465402603149414, 'epsilon_dpo/beta_margin_grad_mean': -0.360959529876709, 'epsilon_dpo/beta_margin_grad_std': 0.11504260450601578, 'kl/beta': 0.0020599286071956158, 'kl/avg_steps': 0.78125, 'epoch': 0.72}
|
|||
|
|
72%|██████████████████████████████████████████████████████████████████████████████████▎ | 492/681 [32:07<07:47, 2.47s/it]
72%|██████████████████████████████████████████████████████████████████████████████████▌ | 493/681 [32:09<07:52, 2.51s/it]
{'loss': 0.9167, 'grad_norm': 38.02676773071289, 'learning_rate': 1.0871589639435203e-07, 'rewards/chosen': -0.3943876028060913, 'rewards/rejected': -1.0185136795043945, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.6241260766983032, 'logps/chosen': -245.80068969726562, 'logps/rejected': -593.231201171875, 'logps/ref_chosen': -52.45185089111328, 'logps/ref_rejected': -91.2623291015625, 'logits/chosen': -2.806032657623291, 'logits/rejected': -3.0264089107513428, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0020301109179854393, 'epsilon_dpo/loss_margin_mean': 308.62005615234375, 'epsilon_dpo/beta_margin_mean': 0.6241260170936584, 'epsilon_dpo/beta_margin_std': 0.5125545859336853, 'epsilon_dpo/beta_margin_grad_mean': -0.3570352792739868, 'epsilon_dpo/beta_margin_grad_std': 0.11298612505197525, 'kl/beta': 0.0020439601503312588, 'kl/avg_steps': 0.6875, 'epoch': 0.72}
|
|||
|
|
72%|██████████████████████████████████████████████████████████████████████████████████▌ | 493/681 [32:09<07:52, 2.51s/it]
73%|██████████████████████████████████████████████████████████████████████████████████▋ | 494/681 [32:12<07:44, 2.48s/it]
{'loss': 0.8385, 'grad_norm': 42.489864349365234, 'learning_rate': 1.0765901824467166e-07, 'rewards/chosen': -0.28744399547576904, 'rewards/rejected': -1.0603857040405273, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.7729417085647583, 'logps/chosen': -170.007568359375, 'logps/rejected': -619.1396484375, 'logps/ref_chosen': -27.903043746948242, 'logps/ref_rejected': -92.12089538574219, 'logits/chosen': -2.750584125518799, 'logits/rejected': -3.075985908508301, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0020137112587690353, 'epsilon_dpo/loss_margin_mean': 384.9142150878906, 'epsilon_dpo/beta_margin_mean': 0.7729417085647583, 'epsilon_dpo/beta_margin_std': 0.6120246052742004, 'epsilon_dpo/beta_margin_grad_mean': -0.3272671103477478, 'epsilon_dpo/beta_margin_grad_std': 0.12577760219573975, 'kl/beta': 0.0020300038158893585, 'kl/avg_steps': 0.8125, 'epoch': 0.73}
|
|||
|
|
73%|██████████████████████████████████████████████████████████████████████████████████▋ | 494/681 [32:12<07:44, 2.48s/it]
73%|██████████████████████████████████████████████████████████████████████████████████▊ | 495/681 [32:14<07:47, 2.51s/it]
{'loss': 0.8, 'grad_norm': 40.98225021362305, 'learning_rate': 1.0660589091223854e-07, 'rewards/chosen': -0.3492244780063629, 'rewards/rejected': -1.1413087844848633, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.7920843362808228, 'logps/chosen': -212.49075317382812, 'logps/rejected': -669.8682861328125, 'logps/ref_chosen': -37.603515625, 'logps/ref_rejected': -97.60113525390625, 'logits/chosen': -2.7651305198669434, 'logits/rejected': -3.138665199279785, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.0019955940078943968, 'epsilon_dpo/loss_margin_mean': 397.37994384765625, 'epsilon_dpo/beta_margin_mean': 0.7920843362808228, 'epsilon_dpo/beta_margin_std': 0.502342939376831, 'epsilon_dpo/beta_margin_grad_mean': -0.3212648630142212, 'epsilon_dpo/beta_margin_grad_std': 0.1038970947265625, 'kl/beta': 0.0020136430393904448, 'kl/avg_steps': 0.90625, 'epoch': 0.73}
|
|||
|
|
73%|██████████████████████████████████████████████████████████████████████████████████▊ | 495/681 [32:14<07:47, 2.51s/it]
73%|███████████████████████████████████████████████████████████████████████████████████ | 496/681 [32:17<07:48, 2.53s/it]
{'loss': 0.8456, 'grad_norm': 36.258426666259766, 'learning_rate': 1.0555654214793722e-07, 'rewards/chosen': -0.3781554102897644, 'rewards/rejected': -1.0874574184417725, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.7093019485473633, 'logps/chosen': -235.90985107421875, 'logps/rejected': -641.947998046875, 'logps/ref_chosen': -45.088035583496094, 'logps/ref_rejected': -92.02516174316406, 'logits/chosen': -2.80366587638855, 'logits/rejected': -3.1190853118896484, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.001978294923901558, 'epsilon_dpo/loss_margin_mean': 359.1010437011719, 'epsilon_dpo/beta_margin_mean': 0.7093020081520081, 'epsilon_dpo/beta_margin_std': 0.45665252208709717, 'epsilon_dpo/beta_margin_grad_mean': -0.3371340036392212, 'epsilon_dpo/beta_margin_grad_std': 0.09828732907772064, 'kl/beta': 0.001995558151975274, 'kl/avg_steps': 0.875, 'epoch': 0.73}
|
|||
|
|
73%|███████████████████████████████████████████████████████████████████████████████████ | 496/681 [32:17<07:48, 2.53s/it]
73%|███████████████████████████████████████████████████████████████████████████████████▏ | 497/681 [32:19<07:40, 2.50s/it]
{'loss': 0.8518, 'grad_norm': 31.69774627685547, 'learning_rate': 1.0451099960308374e-07, 'rewards/chosen': -0.2986891269683838, 'rewards/rejected': -1.003806471824646, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.7051173448562622, 'logps/chosen': -182.076416015625, 'logps/rejected': -593.888671875, 'logps/ref_chosen': -30.02985382080078, 'logps/ref_rejected': -81.73121643066406, 'logits/chosen': -2.7518420219421387, 'logits/rejected': -3.1248888969421387, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.001961135072633624, 'epsilon_dpo/loss_margin_mean': 360.11083984375, 'epsilon_dpo/beta_margin_mean': 0.7051173448562622, 'epsilon_dpo/beta_margin_std': 0.4776591360569, 'epsilon_dpo/beta_margin_grad_mean': -0.3387848436832428, 'epsilon_dpo/beta_margin_grad_std': 0.10064252465963364, 'kl/beta': 0.001978248590603471, 'kl/avg_steps': 0.875, 'epoch': 0.73}
|
|||
|
|
73%|███████████████████████████████████████████████████████████████████████████████████▏ | 497/681 [32:19<07:40, 2.50s/it]
73%|███████████████████████████████████████████████████████████████████████████████████▎ | 498/681 [32:22<07:42, 2.53s/it]
{'loss': 0.9234, 'grad_norm': 43.087093353271484, 'learning_rate': 1.0346929082869641e-07, 'rewards/chosen': -0.40171539783477783, 'rewards/rejected': -1.0523879528045654, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.6506726741790771, 'logps/chosen': -252.92184448242188, 'logps/rejected': -630.257080078125, 'logps/ref_chosen': -47.59989929199219, 'logps/ref_rejected': -89.41059875488281, 'logits/chosen': -2.85072660446167, 'logits/rejected': -3.1245601177215576, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0019471884006634355, 'epsilon_dpo/loss_margin_mean': 335.5245056152344, 'epsilon_dpo/beta_margin_mean': 0.6506726145744324, 'epsilon_dpo/beta_margin_std': 0.6256114840507507, 'epsilon_dpo/beta_margin_grad_mean': -0.35530975461006165, 'epsilon_dpo/beta_margin_grad_std': 0.13022439181804657, 'kl/beta': 0.0019610889721661806, 'kl/avg_steps': 0.71875, 'epoch': 0.73}
|
|||
|
|
73%|███████████████████████████████████████████████████████████████████████████████████▎ | 498/681 [32:22<07:42, 2.53s/it]
73%|███████████████████████████████████████████████████████████████████████████████████▌ | 499/681 [32:25<07:44, 2.55s/it]
{'loss': 0.849, 'grad_norm': 35.18621826171875, 'learning_rate': 1.0243144327477013e-07, 'rewards/chosen': -0.3641180992126465, 'rewards/rejected': -1.1295077800750732, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.765389621257782, 'logps/chosen': -222.212158203125, 'logps/rejected': -693.384033203125, 'logps/ref_chosen': -34.13922882080078, 'logps/ref_rejected': -108.009521484375, 'logits/chosen': -2.809879779815674, 'logits/rejected': -3.214648485183716, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0019314672099426389, 'epsilon_dpo/loss_margin_mean': 397.3015441894531, 'epsilon_dpo/beta_margin_mean': 0.765389621257782, 'epsilon_dpo/beta_margin_std': 0.637798011302948, 'epsilon_dpo/beta_margin_grad_mean': -0.3319936990737915, 'epsilon_dpo/beta_margin_grad_std': 0.13098543882369995, 'kl/beta': 0.0019470942206680775, 'kl/avg_steps': 0.8125, 'epoch': 0.73}
|
|||
|
|
73%|███████████████████████████████████████████████████████████████████████████████████▌ | 499/681 [32:25<07:44, 2.55s/it]
73%|███████████████████████████████████████████████████████████████████████████████████▋ | 500/681 [32:27<07:42, 2.55s/it]
{'loss': 0.8394, 'grad_norm': 37.49474334716797, 'learning_rate': 1.0139748428955333e-07, 'rewards/chosen': -0.37080904841423035, 'rewards/rejected': -1.1393522024154663, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.7685431241989136, 'logps/chosen': -230.16627502441406, 'logps/rejected': -695.9005737304688, 'logps/ref_chosen': -36.92897033691406, 'logps/ref_rejected': -100.48208618164062, 'logits/chosen': -2.810401201248169, 'logits/rejected': -3.235410451889038, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0019146933918818831, 'epsilon_dpo/loss_margin_mean': 402.1811828613281, 'epsilon_dpo/beta_margin_mean': 0.7685431241989136, 'epsilon_dpo/beta_margin_std': 0.6095584630966187, 'epsilon_dpo/beta_margin_grad_mean': -0.3297097384929657, 'epsilon_dpo/beta_margin_grad_std': 0.12436074763536453, 'kl/beta': 0.0019314016681164503, 'kl/avg_steps': 0.875, 'epoch': 0.73}
|
|||
|
|
73%|███████████████████████████████████████████████████████████████████████████████████▋ | 500/681 [32:27<07:42, 2.55s/it][INFO|trainer.py:4307] 2026-04-18 10:03:49,530 >>
|
|||
|
|
***** Running Evaluation *****
|
|||
|
|
[INFO|trainer.py:4309] 2026-04-18 10:03:49,531 >> Num examples = 2339
|
|||
|
|
[INFO|trainer.py:4312] 2026-04-18 10:03:49,531 >> Batch size = 8
|
|||
|
|
|
|||
|
|
0%| | 0/73 [00:00<?, ?it/s][A
|
|||
|
|
3%|███▏ | 2/73 [00:00<00:18, 3.86it/s][A
|
|||
|
|
4%|████▊ | 3/73 [00:01<00:26, 2.61it/s][A
|
|||
|
|
5%|██████▍ | 4/73 [00:01<00:30, 2.26it/s][A
|
|||
|
|
7%|████████ | 5/73 [00:02<00:32, 2.09it/s][A
|
|||
|
|
8%|█████████▌ | 6/73 [00:02<00:33, 2.02it/s][A
|
|||
|
|
10%|███████████▏ | 7/73 [00:03<00:32, 2.01it/s][A
|
|||
|
|
11%|████████████▊ | 8/73 [00:03<00:31, 2.06it/s][A
|
|||
|
|
12%|██████████████▍ | 9/73 [00:04<00:32, 1.99it/s][A
|
|||
|
|
14%|███████████████▉ | 10/73 [00:04<00:32, 1.95it/s][A
|
|||
|
|
15%|█████████████████▍ | 11/73 [00:05<00:31, 1.96it/s][A
|
|||
|
|
16%|███████████████████ | 12/73 [00:05<00:31, 1.96it/s][A
|
|||
|
|
18%|████████████████████▋ | 13/73 [00:06<00:31, 1.91it/s][A
|
|||
|
|
19%|██████████████████████▏ | 14/73 [00:06<00:31, 1.90it/s][A
|
|||
|
|
21%|███████████████████████▊ | 15/73 [00:07<00:31, 1.87it/s][A
|
|||
|
|
22%|█████████████████████████▍ | 16/73 [00:07<00:30, 1.85it/s][A
|
|||
|
|
23%|███████████████████████████ | 17/73 [00:08<00:29, 1.88it/s][A
|
|||
|
|
25%|████████████████████████████▌ | 18/73 [00:09<00:29, 1.84it/s][A
|
|||
|
|
26%|██████████████████████████████▏ | 19/73 [00:09<00:29, 1.84it/s][A
|
|||
|
|
27%|███████████████████████████████▊ | 20/73 [00:10<00:29, 1.83it/s][A
|
|||
|
|
29%|█████████████████████████████████▎ | 21/73 [00:10<00:28, 1.82it/s][A
|
|||
|
|
30%|██████████████████████████████████▉ | 22/73 [00:11<00:27, 1.82it/s][A
|
|||
|
|
32%|████████████████████████████████████▌ | 23/73 [00:11<00:27, 1.84it/s][A
|
|||
|
|
33%|██████████████████████████████████████▏ | 24/73 [00:12<00:26, 1.88it/s][A
|
|||
|
|
34%|███████████████████████████████████████▋ | 25/73 [00:12<00:25, 1.87it/s][A
|
|||
|
|
36%|█████████████████████████████████████████▎ | 26/73 [00:13<00:24, 1.90it/s][A
|
|||
|
|
37%|██████████████████████████████████████████▉ | 27/73 [00:13<00:22, 2.05it/s][A
|
|||
|
|
38%|████████████████████████████████████████████▍ | 28/73 [00:14<00:22, 2.04it/s][A
|
|||
|
|
40%|██████████████████████████████████████████████ | 29/73 [00:14<00:21, 2.00it/s][A
|
|||
|
|
41%|███████████████████████████████████████████████▋ | 30/73 [00:15<00:21, 1.97it/s][A
|
|||
|
|
42%|█████████████████████████████████████████████████▎ | 31/73 [00:15<00:21, 1.92it/s][A
|
|||
|
|
44%|██████████████████████████████████████████████████▊ | 32/73 [00:16<00:21, 1.94it/s][A
|
|||
|
|
45%|████████████████████████████████████████████████████▍ | 33/73 [00:16<00:20, 1.93it/s][A
|
|||
|
|
47%|██████████████████████████████████████████████████████ | 34/73 [00:17<00:19, 1.97it/s][A
|
|||
|
|
48%|███████████████████████████████████████████████████████▌ | 35/73 [00:17<00:19, 1.90it/s][A
|
|||
|
|
49%|█████████████████████████████████████████████████████████▏ | 36/73 [00:18<00:19, 1.92it/s][A
|
|||
|
|
51%|██████████████████████████████████████████████████████████▊ | 37/73 [00:18<00:19, 1.88it/s][A
|
|||
|
|
52%|████████████████████████████████████████████████████████████▍ | 38/73 [00:19<00:18, 1.93it/s][A
|
|||
|
|
53%|█████████████████████████████████████████████████████████████▉ | 39/73 [00:19<00:17, 1.92it/s][A
|
|||
|
|
55%|███████████████████████████████████████████████████████████████▌ | 40/73 [00:20<00:17, 1.92it/s][A
|
|||
|
|
56%|█████████████████████████████████████████████████████████████████▏ | 41/73 [00:20<00:16, 1.94it/s][A
|
|||
|
|
58%|██████████████████████████████████████████████████████████████████▋ | 42/73 [00:21<00:16, 1.90it/s][A
|
|||
|
|
59%|████████████████████████████████████████████████████████████████████▎ | 43/73 [00:22<00:15, 1.88it/s][A
|
|||
|
|
60%|█████████████████████████████████████████████████████████████████████▉ | 44/73 [00:22<00:15, 1.86it/s][A
|
|||
|
|
62%|███████████████████████████████████████████████████████████████████████▌ | 45/73 [00:23<00:15, 1.82it/s][A
|
|||
|
|
63%|█████████████████████████████████████████████████████████████████████████ | 46/73 [00:23<00:14, 1.87it/s][A
|
|||
|
|
64%|██████████████████████████████████████████████████████████████████████████▋ | 47/73 [00:24<00:13, 1.88it/s][A
|
|||
|
|
66%|████████████████████████████████████████████████████████████████████████████▎ | 48/73 [00:24<00:12, 1.93it/s][A
|
|||
|
|
67%|█████████████████████████████████████████████████████████████████████████████▊ | 49/73 [00:25<00:12, 1.88it/s][A
|
|||
|
|
68%|███████████████████████████████████████████████████████████████████████████████▍ | 50/73 [00:25<00:12, 1.86it/s][A
|
|||
|
|
70%|█████████████████████████████████████████████████████████████████████████████████ | 51/73 [00:26<00:12, 1.83it/s][A
|
|||
|
|
71%|██████████████████████████████████████████████████████████████████████████████████▋ | 52/73 [00:26<00:11, 1.81it/s][A
|
|||
|
|
73%|████████████████████████████████████████████████████████████████████████████████████▏ | 53/73 [00:27<00:10, 1.94it/s][A
|
|||
|
|
74%|█████████████████████████████████████████████████████████████████████████████████████▊ | 54/73 [00:27<00:09, 1.96it/s][A
|
|||
|
|
75%|███████████████████████████████████████████████████████████████████████████████████████▍ | 55/73 [00:28<00:08, 2.00it/s][A
|
|||
|
|
77%|████████████████████████████████████████████████████████████████████████████████████████▉ | 56/73 [00:28<00:08, 1.96it/s][A
|
|||
|
|
78%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 57/73 [00:29<00:08, 1.91it/s][A
|
|||
|
|
79%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 58/73 [00:29<00:07, 1.94it/s][A
|
|||
|
|
81%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 59/73 [00:30<00:07, 1.88it/s][A
|
|||
|
|
82%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 60/73 [00:31<00:06, 1.89it/s][A
|
|||
|
|
84%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 61/73 [00:31<00:06, 1.87it/s][A
|
|||
|
|
85%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 62/73 [00:32<00:05, 1.84it/s][A
|
|||
|
|
86%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 63/73 [00:32<00:05, 1.90it/s][A
|
|||
|
|
88%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 64/73 [00:33<00:04, 1.92it/s][A
|
|||
|
|
89%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 65/73 [00:33<00:04, 1.87it/s][A
|
|||
|
|
90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 66/73 [00:34<00:03, 1.86it/s][A
|
|||
|
|
92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 67/73 [00:34<00:03, 1.85it/s][A
|
|||
|
|
93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 68/73 [00:35<00:02, 1.86it/s][A
|
|||
|
|
95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 69/73 [00:35<00:02, 1.97it/s][A
|
|||
|
|
96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 70/73 [00:36<00:01, 1.91it/s][A
|
|||
|
|
97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 71/73 [00:36<00:01, 1.90it/s][A
|
|||
|
|
99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 72/73 [00:37<00:00, 1.88it/s][A
|
|||
|
|
100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 73/73 [00:37<00:00, 1.99it/s][A
|
|||
|
|
[A{'eval_loss': 0.5502132177352905, 'eval_runtime': 38.4009, 'eval_samples_per_second': 60.91, 'eval_steps_per_second': 1.927, 'eval_epsilon_dpo/beta': 0.0019060522317886353, 'eval_epsilon_dpo/loss_margin_mean': 226.7985382080078, 'eval_epsilon_dpo/beta_margin_mean': 0.4280838668346405, 'eval_epsilon_dpo/beta_margin_std': 0.6343588829040527, 'eval_epsilon_dpo/beta_margin_grad_mean': -0.4040687382221222, 'eval_epsilon_dpo/beta_margin_grad_std': 0.1391373723745346, 'eval_rewards/chosen': -0.6045443415641785, 'eval_rewards/rejected': -1.0326281785964966, 'eval_rewards/accuracies': 0.7452911138534546, 'eval_rewards/margins': 0.4280838668346405, 'eval_logps/chosen': -384.0945129394531, 'eval_logps/rejected': -634.682373046875, 'eval_logps/ref_chosen': -68.29110717773438, 'eval_logps/ref_rejected': -92.08038330078125, 'eval_logits/chosen': -2.9733166694641113, 'eval_logits/rejected': -3.179455518722534, 'eval_kl/p_epsilon_steps': 0.7290239930152893, 'eval_kl/n_epsilon_steps': 0.27011987566947937, 'epoch': 0.73}
|
|||
|
|
73%|███████████████████████████████████████████████████████████████████████████████████▋ | 500/681 [33:06<07:42, 2.55s/it]
|
|||
|
|
100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 73/73 [00:37<00:00, 1.99it/s][A
|
|||
|
|
[A
74%|███████████████████████████████████████████████████████████████████████████████████▊ | 501/681 [33:08<42:22, 14.13s/it]
{'loss': 0.8571, 'grad_norm': 35.63896179199219, 'learning_rate': 1.0036744111882672e-07, 'rewards/chosen': -0.35178065299987793, 'rewards/rejected': -1.0915746688842773, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.7397940158843994, 'logps/chosen': -224.27859497070312, 'logps/rejected': -660.603515625, 'logps/ref_chosen': -39.85574722290039, 'logps/ref_rejected': -85.64679718017578, 'logits/chosen': -2.834397077560425, 'logits/rejected': -3.2369680404663086, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0018998802406713367, 'epsilon_dpo/loss_margin_mean': 390.53387451171875, 'epsilon_dpo/beta_margin_mean': 0.7397939562797546, 'epsilon_dpo/beta_margin_std': 0.6008874773979187, 'epsilon_dpo/beta_margin_grad_mean': -0.3355919420719147, 'epsilon_dpo/beta_margin_grad_std': 0.12509720027446747, 'kl/beta': 0.001914648455567658, 'kl/avg_steps': 0.78125, 'epoch': 0.74}
|
|||
|
|
74%|███████████████████████████████████████████████████████████████████████████████████▊ | 501/681 [33:08<42:22, 14.13s/it]
74%|████████████████████████████████████████████████████████████████████████████████████ | 502/681 [33:11<32:02, 10.74s/it]
{'loss': 0.9721, 'grad_norm': 49.43293762207031, 'learning_rate': 9.934134090518592e-08, 'rewards/chosen': -0.41469526290893555, 'rewards/rejected': -0.9710033535957336, 'rewards/accuracies': 0.875, 'rewards/margins': 0.5563080906867981, 'logps/chosen': -272.3970031738281, 'logps/rejected': -602.1082153320312, 'logps/ref_chosen': -53.44129943847656, 'logps/ref_rejected': -86.86759185791016, 'logits/chosen': -2.929020404815674, 'logits/rejected': -3.1855547428131104, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0018857462564483285, 'epsilon_dpo/loss_margin_mean': 296.284912109375, 'epsilon_dpo/beta_margin_mean': 0.5563080906867981, 'epsilon_dpo/beta_margin_std': 0.5363189578056335, 'epsilon_dpo/beta_margin_grad_mean': -0.37188294529914856, 'epsilon_dpo/beta_margin_grad_std': 0.12015336006879807, 'kl/beta': 0.0018998062005266547, 'kl/avg_steps': 0.75, 'epoch': 0.74}
|
|||
|
|
74%|████████████████████████████████████████████████████████████████████████████████████ | 502/681 [33:11<32:02, 10.74s/it]
74%|████████████████████████████████████████████████████████████████████████████████████▏ | 503/681 [33:13<24:27, 8.24s/it]
{'loss': 0.862, 'grad_norm': 37.46735382080078, 'learning_rate': 9.831921068732571e-08, 'rewards/chosen': -0.28525102138519287, 'rewards/rejected': -0.9941678047180176, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.7089167833328247, 'logps/chosen': -186.5693817138672, 'logps/rejected': -620.7432250976562, 'logps/ref_chosen': -34.42343521118164, 'logps/ref_rejected': -88.69686889648438, 'logits/chosen': -2.8521652221679688, 'logits/rejected': -3.250486135482788, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0018699404317885637, 'epsilon_dpo/loss_margin_mean': 379.900390625, 'epsilon_dpo/beta_margin_mean': 0.7089167833328247, 'epsilon_dpo/beta_margin_std': 0.5404109358787537, 'epsilon_dpo/beta_margin_grad_mean': -0.34032920002937317, 'epsilon_dpo/beta_margin_grad_std': 0.11132641136646271, 'kl/beta': 0.0018856637179851532, 'kl/avg_steps': 0.84375, 'epoch': 0.74}
|
|||
|
|
74%|████████████████████████████████████████████████████████████████████████████████████▏ | 503/681 [33:13<24:27, 8.24s/it]
74%|████████████████████████████████████████████████████████████████████████████████████▎ | 504/681 [33:16<19:10, 6.50s/it]
{'loss': 0.8029, 'grad_norm': 43.66539001464844, 'learning_rate': 9.730107739932805e-08, 'rewards/chosen': -0.3029179573059082, 'rewards/rejected': -1.1319552659988403, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.8290372490882874, 'logps/chosen': -209.00950622558594, 'logps/rejected': -721.7181396484375, 'logps/ref_chosen': -46.35227966308594, 'logps/ref_rejected': -111.0133056640625, 'logits/chosen': -2.828281879425049, 'logits/rejected': -3.302220106124878, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0018554636044427752, 'epsilon_dpo/loss_margin_mean': 448.0475769042969, 'epsilon_dpo/beta_margin_mean': 0.8290372490882874, 'epsilon_dpo/beta_margin_std': 0.6291282176971436, 'epsilon_dpo/beta_margin_grad_mean': -0.31814929842948914, 'epsilon_dpo/beta_margin_grad_std': 0.12207160890102386, 'kl/beta': 0.0018698865314945579, 'kl/avg_steps': 0.78125, 'epoch': 0.74}
|
|||
|
|
74%|████████████████████████████████████████████████████████████████████████████████████▎ | 504/681 [33:16<19:10, 6.50s/it]
74%|████████████████████████████████████████████████████████████████████████████████████▌ | 505/681 [33:18<15:33, 5.30s/it]
{'loss': 1.0064, 'grad_norm': 45.315521240234375, 'learning_rate': 9.628696786995188e-08, 'rewards/chosen': -0.39312461018562317, 'rewards/rejected': -0.9640980958938599, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.5709735155105591, 'logps/chosen': -259.9178771972656, 'logps/rejected': -620.2982177734375, 'logps/ref_chosen': -47.89013671875, 'logps/ref_rejected': -96.4730453491211, 'logits/chosen': -2.8761417865753174, 'logits/rejected': -3.2082276344299316, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.0018439794657751918, 'epsilon_dpo/loss_margin_mean': 311.79742431640625, 'epsilon_dpo/beta_margin_mean': 0.5709735155105591, 'epsilon_dpo/beta_margin_std': 0.7030720710754395, 'epsilon_dpo/beta_margin_grad_mean': -0.372370183467865, 'epsilon_dpo/beta_margin_grad_std': 0.15154238045215607, 'kl/beta': 0.0018553913105279207, 'kl/avg_steps': 0.625, 'epoch': 0.74}
|
|||
|
|
74%|████████████████████████████████████████████████████████████████████████████████████▌ | 505/681 [33:18<15:33, 5.30s/it]
74%|████████████████████████████████████████████████████████████████████████████████████▋ | 506/681 [33:21<12:58, 4.45s/it]
{'loss': 0.8885, 'grad_norm': 39.87546920776367, 'learning_rate': 9.527690882192635e-08, 'rewards/chosen': -0.3329414129257202, 'rewards/rejected': -1.0024747848510742, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.669533371925354, 'logps/chosen': -212.9695281982422, 'logps/rejected': -633.22314453125, 'logps/ref_chosen': -31.555944442749023, 'logps/ref_rejected': -84.6425552368164, 'logits/chosen': -2.8096556663513184, 'logits/rejected': -3.270047187805176, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0018290685256943107, 'epsilon_dpo/loss_margin_mean': 367.1669616699219, 'epsilon_dpo/beta_margin_mean': 0.6695333123207092, 'epsilon_dpo/beta_margin_std': 0.5264294743537903, 'epsilon_dpo/beta_margin_grad_mean': -0.3460908532142639, 'epsilon_dpo/beta_margin_grad_std': 0.11466103047132492, 'kl/beta': 0.0018438671249896288, 'kl/avg_steps': 0.8125, 'epoch': 0.74}
|
|||
|
|
74%|████████████████████████████████████████████████████████████████████████████████████▋ | 506/681 [33:21<12:58, 4.45s/it]
74%|████████████████████████████████████████████████████████████████████████████████████▊ | 507/681 [33:23<11:13, 3.87s/it]
{'loss': 0.9498, 'grad_norm': 34.283470153808594, 'learning_rate': 9.427092687124691e-08, 'rewards/chosen': -0.42130210995674133, 'rewards/rejected': -1.009988784790039, 'rewards/accuracies': 0.875, 'rewards/margins': 0.5886867642402649, 'logps/chosen': -282.3764953613281, 'logps/rejected': -658.5035400390625, 'logps/ref_chosen': -50.91284942626953, 'logps/ref_rejected': -101.67366790771484, 'logits/chosen': -2.9394993782043457, 'logits/rejected': -3.2683815956115723, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.001816042116843164, 'epsilon_dpo/loss_margin_mean': 325.3662414550781, 'epsilon_dpo/beta_margin_mean': 0.5886867642402649, 'epsilon_dpo/beta_margin_std': 0.5481820702552795, 'epsilon_dpo/beta_margin_grad_mean': -0.366396963596344, 'epsilon_dpo/beta_margin_grad_std': 0.11917287856340408, 'kl/beta': 0.001829006476327777, 'kl/avg_steps': 0.71875, 'epoch': 0.74}
|
|||
|
|
74%|████████████████████████████████████████████████████████████████████████████████████▊ | 507/681 [33:23<11:13, 3.87s/it]
75%|█████████████████████████████████████████████████████████████████████████████████████ | 508/681 [33:26<10:06, 3.50s/it]
{'loss': 0.9286, 'grad_norm': 40.939327239990234, 'learning_rate': 9.326904852647344e-08, 'rewards/chosen': -0.38065105676651, 'rewards/rejected': -1.0160696506500244, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.6354185342788696, 'logps/chosen': -260.6716613769531, 'logps/rejected': -665.6597900390625, 'logps/ref_chosen': -50.73012924194336, 'logps/ref_rejected': -101.8434829711914, 'logits/chosen': -2.883289337158203, 'logits/rejected': -3.2906653881073, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.0018047851044684649, 'epsilon_dpo/loss_margin_mean': 353.8747863769531, 'epsilon_dpo/beta_margin_mean': 0.6354185342788696, 'epsilon_dpo/beta_margin_std': 0.6018913984298706, 'epsilon_dpo/beta_margin_grad_mean': -0.3578824996948242, 'epsilon_dpo/beta_margin_grad_std': 0.12713228166103363, 'kl/beta': 0.0018159543396905065, 'kl/avg_steps': 0.625, 'epoch': 0.75}
|
|||
|
|
75%|█████████████████████████████████████████████████████████████████████████████████████ | 508/681 [33:26<10:06, 3.50s/it]
75%|█████████████████████████████████████████████████████████████████████████████████████▏ | 509/681 [33:29<09:17, 3.24s/it]
{'loss': 0.8974, 'grad_norm': 38.303802490234375, 'learning_rate': 9.227130018803195e-08, 'rewards/chosen': -0.30018889904022217, 'rewards/rejected': -0.9426673650741577, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.6424784660339355, 'logps/chosen': -214.93182373046875, 'logps/rejected': -615.6307373046875, 'logps/ref_chosen': -47.68657302856445, 'logps/ref_rejected': -88.62358093261719, 'logits/chosen': -2.86599063873291, 'logits/rejected': -3.2632508277893066, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0017901910468935966, 'epsilon_dpo/loss_margin_mean': 359.7619323730469, 'epsilon_dpo/beta_margin_mean': 0.6424784660339355, 'epsilon_dpo/beta_margin_std': 0.4860381782054901, 'epsilon_dpo/beta_margin_grad_mean': -0.35225531458854675, 'epsilon_dpo/beta_margin_grad_std': 0.10537883639335632, 'kl/beta': 0.0018046750919893384, 'kl/avg_steps': 0.8125, 'epoch': 0.75}
|
|||
|
|
75%|█████████████████████████████████████████████████████████████████████████████████████▏ | 509/681 [33:29<09:17, 3.24s/it]
75%|█████████████████████████████████████████████████████████████████████████████████████▎ | 510/681 [33:31<08:38, 3.03s/it]
{'loss': 0.8498, 'grad_norm': 32.25279235839844, 'learning_rate': 9.127770814751932e-08, 'rewards/chosen': -0.33473625779151917, 'rewards/rejected': -1.043351650238037, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.7086154222488403, 'logps/chosen': -220.7513427734375, 'logps/rejected': -691.013427734375, 'logps/ref_chosen': -32.308326721191406, 'logps/ref_rejected': -102.69437408447266, 'logits/chosen': -2.808089256286621, 'logits/rejected': -3.3828306198120117, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.001774644129909575, 'epsilon_dpo/loss_margin_mean': 399.8760070800781, 'epsilon_dpo/beta_margin_mean': 0.7086153626441956, 'epsilon_dpo/beta_margin_std': 0.4846038222312927, 'epsilon_dpo/beta_margin_grad_mean': -0.33876028656959534, 'epsilon_dpo/beta_margin_grad_std': 0.0986710786819458, 'kl/beta': 0.001790130278095603, 'kl/avg_steps': 0.875, 'epoch': 0.75}
|
|||
|
|
75%|█████████████████████████████████████████████████████████████████████████████████████▎ | 510/681 [33:31<08:38, 3.03s/it]
75%|█████████████████████████████████████████████████████████████████████████████████████▌ | 511/681 [33:34<08:09, 2.88s/it]
{'loss': 0.8819, 'grad_norm': 34.81239318847656, 'learning_rate': 9.028829858700973e-08, 'rewards/chosen': -0.3278487026691437, 'rewards/rejected': -0.9970583915710449, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.6692096590995789, 'logps/chosen': -226.4573974609375, 'logps/rejected': -665.3365478515625, 'logps/ref_chosen': -41.02062225341797, 'logps/ref_rejected': -98.91937255859375, 'logits/chosen': -2.88916015625, 'logits/rejected': -3.333850860595703, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0017620236612856388, 'epsilon_dpo/loss_margin_mean': 380.9803466796875, 'epsilon_dpo/beta_margin_mean': 0.6692096590995789, 'epsilon_dpo/beta_margin_std': 0.5043637752532959, 'epsilon_dpo/beta_margin_grad_mean': -0.34761884808540344, 'epsilon_dpo/beta_margin_grad_std': 0.10665444284677505, 'kl/beta': 0.001774602453224361, 'kl/avg_steps': 0.71875, 'epoch': 0.75}
|
|||
|
|
75%|█████████████████████████████████████████████████████████████████████████████████████▌ | 511/681 [33:34<08:09, 2.88s/it]
75%|█████████████████████████████████████████████████████████████████████████████████████▋ | 512/681 [33:36<07:40, 2.72s/it]
{'loss': 0.8953, 'grad_norm': 34.644046783447266, 'learning_rate': 8.930309757836516e-08, 'rewards/chosen': -0.30911028385162354, 'rewards/rejected': -0.9897146224975586, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.6806043386459351, 'logps/chosen': -213.54293823242188, 'logps/rejected': -654.92138671875, 'logps/ref_chosen': -37.26538848876953, 'logps/ref_rejected': -88.17929077148438, 'logits/chosen': -2.82757306098938, 'logits/rejected': -3.3794026374816895, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.001748348237015307, 'epsilon_dpo/loss_margin_mean': 390.4645690917969, 'epsilon_dpo/beta_margin_mean': 0.6806043386459351, 'epsilon_dpo/beta_margin_std': 0.5956137180328369, 'epsilon_dpo/beta_margin_grad_mean': -0.3473552465438843, 'epsilon_dpo/beta_margin_grad_std': 0.12345966696739197, 'kl/beta': 0.0017619385616853833, 'kl/avg_steps': 0.78125, 'epoch': 0.75}
|
|||
|
|
75%|█████████████████████████████████████████████████████████████████████████████████████▋ | 512/681 [33:36<07:40, 2.72s/it]
75%|█████████████████████████████████████████████████████████████████████████████████████▉ | 513/681 [33:39<07:41, 2.75s/it]
{'loss': 0.9234, 'grad_norm': 37.334041595458984, 'learning_rate': 8.832213108254863e-08, 'rewards/chosen': -0.2673969268798828, 'rewards/rejected': -0.8836780786514282, 'rewards/accuracies': 0.875, 'rewards/margins': 0.6162811517715454, 'logps/chosen': -189.08787536621094, 'logps/rejected': -591.59619140625, 'logps/ref_chosen': -35.62997055053711, 'logps/ref_rejected': -81.91015625, 'logits/chosen': -2.8295106887817383, 'logits/rejected': -3.2998032569885254, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0017353416187688708, 'epsilon_dpo/loss_margin_mean': 356.2281494140625, 'epsilon_dpo/beta_margin_mean': 0.6162811517715454, 'epsilon_dpo/beta_margin_std': 0.5283299684524536, 'epsilon_dpo/beta_margin_grad_mean': -0.3601101040840149, 'epsilon_dpo/beta_margin_grad_std': 0.1102689802646637, 'kl/beta': 0.0017482801340520382, 'kl/avg_steps': 0.75, 'epoch': 0.75}
|
|||
|
|
75%|█████████████████████████████████████████████████████████████████████████████████████▉ | 513/681 [33:39<07:41, 2.75s/it]
75%|██████████████████████████████████████████████████████████████████████████████████████ | 514/681 [33:41<07:24, 2.66s/it]
{'loss': 0.9222, 'grad_norm': 29.60846519470215, 'learning_rate': 8.734542494893954e-08, 'rewards/chosen': -0.3422423005104065, 'rewards/rejected': -0.969772219657898, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.6275299787521362, 'logps/chosen': -241.52822875976562, 'logps/rejected': -648.3846435546875, 'logps/ref_chosen': -43.213531494140625, 'logps/ref_rejected': -84.48947143554688, 'logits/chosen': -2.9027023315429688, 'logits/rejected': -3.4183146953582764, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0017213387181982398, 'epsilon_dpo/loss_margin_mean': 365.5804748535156, 'epsilon_dpo/beta_margin_mean': 0.6275299191474915, 'epsilon_dpo/beta_margin_std': 0.5512734055519104, 'epsilon_dpo/beta_margin_grad_mean': -0.35787054896354675, 'epsilon_dpo/beta_margin_grad_std': 0.11788881570100784, 'kl/beta': 0.0017352655995637178, 'kl/avg_steps': 0.8125, 'epoch': 0.75}
|
|||
|
|
75%|██████████████████████████████████████████████████████████████████████████████████████ | 514/681 [33:41<07:24, 2.66s/it]
76%|██████████████████████████████████████████████████████████████████████████████████████▏ | 515/681 [33:44<07:18, 2.64s/it]
{'loss': 0.9554, 'grad_norm': 36.903953552246094, 'learning_rate': 8.637300491465272e-08, 'rewards/chosen': -0.34564244747161865, 'rewards/rejected': -0.9458876848220825, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.6002452969551086, 'logps/chosen': -235.07046508789062, 'logps/rejected': -646.9464111328125, 'logps/ref_chosen': -33.94335174560547, 'logps/ref_rejected': -92.82087707519531, 'logits/chosen': -2.862236738204956, 'logits/rejected': -3.4654173851013184, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0017085415311157703, 'epsilon_dpo/loss_margin_mean': 352.9983825683594, 'epsilon_dpo/beta_margin_mean': 0.6002452969551086, 'epsilon_dpo/beta_margin_std': 0.5947220921516418, 'epsilon_dpo/beta_margin_grad_mean': -0.3623103201389313, 'epsilon_dpo/beta_margin_grad_std': 0.13232813775539398, 'kl/beta': 0.0017212802777066827, 'kl/avg_steps': 0.75, 'epoch': 0.76}
|
|||
|
|
76%|██████████████████████████████████████████████████████████████████████████████████████▏ | 515/681 [33:44<07:18, 2.64s/it]
76%|██████████████████████████████████████████████████████████████████████████████████████▍ | 516/681 [33:46<07:02, 2.56s/it]
{'loss': 0.8964, 'grad_norm': 35.10322952270508, 'learning_rate': 8.540489660386064e-08, 'rewards/chosen': -0.33476826548576355, 'rewards/rejected': -1.0319504737854004, 'rewards/accuracies': 0.875, 'rewards/margins': 0.6971822381019592, 'logps/chosen': -244.57504272460938, 'logps/rejected': -728.8509521484375, 'logps/ref_chosen': -48.20185852050781, 'logps/ref_rejected': -119.76963806152344, 'logits/chosen': -2.9307332038879395, 'logits/rejected': -3.505221366882324, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0016963566886261106, 'epsilon_dpo/loss_margin_mean': 412.7080993652344, 'epsilon_dpo/beta_margin_mean': 0.6971822381019592, 'epsilon_dpo/beta_margin_std': 0.6565114259719849, 'epsilon_dpo/beta_margin_grad_mean': -0.3456820547580719, 'epsilon_dpo/beta_margin_grad_std': 0.12934663891792297, 'kl/beta': 0.0017084666760638356, 'kl/avg_steps': 0.71875, 'epoch': 0.76}
|
|||
|
|
76%|██████████████████████████████████████████████████████████████████████████████████████▍ | 516/681 [33:46<07:02, 2.56s/it]
76%|██████████████████████████████████████████████████████████████████████████████████████▌ | 517/681 [33:49<06:58, 2.55s/it]
{'loss': 0.9048, 'grad_norm': 38.30036163330078, 'learning_rate': 8.444112552711752e-08, 'rewards/chosen': -0.33436378836631775, 'rewards/rejected': -1.0052188634872437, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.6708550453186035, 'logps/chosen': -237.81472778320312, 'logps/rejected': -696.3298950195312, 'logps/ref_chosen': -40.42920684814453, 'logps/ref_rejected': -98.80826568603516, 'logits/chosen': -2.9165990352630615, 'logits/rejected': -3.483713150024414, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0016842512413859367, 'epsilon_dpo/loss_margin_mean': 400.1361083984375, 'epsilon_dpo/beta_margin_mean': 0.6708550453186035, 'epsilon_dpo/beta_margin_std': 0.5967744588851929, 'epsilon_dpo/beta_margin_grad_mean': -0.3475935757160187, 'epsilon_dpo/beta_margin_grad_std': 0.12928926944732666, 'kl/beta': 0.0016962747322395444, 'kl/avg_steps': 0.71875, 'epoch': 0.76}
|
|||
|
|
76%|██████████████████████████████████████████████████████████████████████████████████████▌ | 517/681 [33:49<06:58, 2.55s/it]
76%|██████████████████████████████████████████████████████████████████████████████████████▋ | 518/681 [33:51<06:55, 2.55s/it]
{'loss': 0.9249, 'grad_norm': 43.182456970214844, 'learning_rate': 8.348171708068747e-08, 'rewards/chosen': -0.3209276795387268, 'rewards/rejected': -0.9497312307357788, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.6288034915924072, 'logps/chosen': -228.24789428710938, 'logps/rejected': -659.341552734375, 'logps/ref_chosen': -37.2679557800293, 'logps/ref_rejected': -90.86311340332031, 'logits/chosen': -2.878077983856201, 'logits/rejected': -3.482807159423828, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0016727583715692163, 'epsilon_dpo/loss_margin_mean': 377.4985046386719, 'epsilon_dpo/beta_margin_mean': 0.6288034915924072, 'epsilon_dpo/beta_margin_std': 0.5658082962036133, 'epsilon_dpo/beta_margin_grad_mean': -0.35805386304855347, 'epsilon_dpo/beta_margin_grad_std': 0.1211152896285057, 'kl/beta': 0.0016841697506606579, 'kl/avg_steps': 0.6875, 'epoch': 0.76}
|
|||
|
|
76%|██████████████████████████████████████████████████████████████████████████████████████▋ | 518/681 [33:51<06:55, 2.55s/it]
76%|██████████████████████████████████████████████████████████████████████████████████████▉ | 519/681 [33:54<06:56, 2.57s/it]
{'loss': 0.9302, 'grad_norm': 37.42156982421875, 'learning_rate': 8.25266965458755e-08, 'rewards/chosen': -0.32136669754981995, 'rewards/rejected': -0.9656132459640503, 'rewards/accuracies': 0.875, 'rewards/margins': 0.6442465782165527, 'logps/chosen': -240.69334411621094, 'logps/rejected': -693.385009765625, 'logps/ref_chosen': -47.603675842285156, 'logps/ref_rejected': -110.82022094726562, 'logits/chosen': -2.999945878982544, 'logits/rejected': -3.565791606903076, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0016592455795034766, 'epsilon_dpo/loss_margin_mean': 389.47509765625, 'epsilon_dpo/beta_margin_mean': 0.644246518611908, 'epsilon_dpo/beta_margin_std': 0.639655590057373, 'epsilon_dpo/beta_margin_grad_mean': -0.35770776867866516, 'epsilon_dpo/beta_margin_grad_std': 0.13007250428199768, 'kl/beta': 0.0016726701287552714, 'kl/avg_steps': 0.8125, 'epoch': 0.76}
|
|||
|
|
76%|██████████████████████████████████████████████████████████████████████████████████████▉ | 519/681 [33:54<06:56, 2.57s/it]
76%|███████████████████████████████████████████████████████████████████████████████████████ | 520/681 [33:57<06:56, 2.59s/it]
{'loss': 0.8746, 'grad_norm': 30.94179916381836, 'learning_rate': 8.15760890883607e-08, 'rewards/chosen': -0.3073316216468811, 'rewards/rejected': -0.9924643039703369, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.685132622718811, 'logps/chosen': -229.61695861816406, 'logps/rejected': -712.6129150390625, 'logps/ref_chosen': -43.462921142578125, 'logps/ref_rejected': -109.22837829589844, 'logits/chosen': -2.9207892417907715, 'logits/rejected': -3.5509166717529297, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0016458729514852166, 'epsilon_dpo/loss_margin_mean': 417.23046875, 'epsilon_dpo/beta_margin_mean': 0.685132622718811, 'epsilon_dpo/beta_margin_std': 0.5305820107460022, 'epsilon_dpo/beta_margin_grad_mean': -0.3453068435192108, 'epsilon_dpo/beta_margin_grad_std': 0.10654862225055695, 'kl/beta': 0.0016591892344877124, 'kl/avg_steps': 0.8125, 'epoch': 0.76}
|
|||
|
|
76%|███████████████████████████████████████████████████████████████████████████████████████ | 520/681 [33:57<06:56, 2.59s/it]
77%|███████████████████████████████████████████████████████████████████████████████████████▏ | 521/681 [33:59<06:51, 2.57s/it]
{'loss': 0.9344, 'grad_norm': 37.41933059692383, 'learning_rate': 8.062991975753378e-08, 'rewards/chosen': -0.2921355366706848, 'rewards/rejected': -0.8830931782722473, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.5909576416015625, 'logps/chosen': -218.08642578125, 'logps/rejected': -630.3876953125, 'logps/ref_chosen': -39.79935836791992, 'logps/ref_rejected': -89.2203140258789, 'logits/chosen': -2.949124336242676, 'logits/rejected': -3.4791650772094727, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.001633122330531478, 'epsilon_dpo/loss_margin_mean': 362.88031005859375, 'epsilon_dpo/beta_margin_mean': 0.5909576416015625, 'epsilon_dpo/beta_margin_std': 0.4859505593776703, 'epsilon_dpo/beta_margin_grad_mean': -0.3637232482433319, 'epsilon_dpo/beta_margin_grad_std': 0.10722323507070541, 'kl/beta': 0.0016458169557154179, 'kl/avg_steps': 0.78125, 'epoch': 0.77}
|
|||
|
|
77%|███████████████████████████████████████████████████████████████████████████████████████▏ | 521/681 [33:59<06:51, 2.57s/it]
77%|███████████████████████████████████████████████████████████████████████████████████████▍ | 522/681 [34:02<06:42, 2.53s/it]
{'loss': 1.0096, 'grad_norm': 34.424713134765625, 'learning_rate': 7.968821348583643e-08, 'rewards/chosen': -0.3123043179512024, 'rewards/rejected': -0.8234931230545044, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.511188805103302, 'logps/chosen': -223.6859893798828, 'logps/rejected': -581.035888671875, 'logps/ref_chosen': -32.023380279541016, 'logps/ref_rejected': -72.67029571533203, 'logits/chosen': -2.922499895095825, 'logits/rejected': -3.3887128829956055, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0016209728782996535, 'epsilon_dpo/loss_margin_mean': 316.7029724121094, 'epsilon_dpo/beta_margin_mean': 0.511188805103302, 'epsilon_dpo/beta_margin_std': 0.5572635531425476, 'epsilon_dpo/beta_margin_grad_mean': -0.38271984457969666, 'epsilon_dpo/beta_margin_grad_std': 0.12090186774730682, 'kl/beta': 0.0016330587677657604, 'kl/avg_steps': 0.75, 'epoch': 0.77}
|
|||
|
|
77%|███████████████████████████████████████████████████████████████████████████████████████▍ | 522/681 [34:02<06:42, 2.53s/it]
77%|███████████████████████████████████████████████████████████████████████████████████████▌ | 523/681 [34:04<06:36, 2.51s/it]
{'loss': 0.9538, 'grad_norm': 34.04196548461914, 'learning_rate': 7.875099508810484e-08, 'rewards/chosen': -0.2807372212409973, 'rewards/rejected': -0.8746113777160645, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.5938742160797119, 'logps/chosen': -218.07928466796875, 'logps/rejected': -637.406982421875, 'logps/ref_chosen': -44.56959533691406, 'logps/ref_rejected': -93.40034484863281, 'logits/chosen': -2.9228086471557617, 'logits/rejected': -3.478931427001953, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.0016109324060380459, 'epsilon_dpo/loss_margin_mean': 370.4969482421875, 'epsilon_dpo/beta_margin_mean': 0.5938741564750671, 'epsilon_dpo/beta_margin_std': 0.5870850682258606, 'epsilon_dpo/beta_margin_grad_mean': -0.36688175797462463, 'epsilon_dpo/beta_margin_grad_std': 0.12298644334077835, 'kl/beta': 0.001620901981368661, 'kl/avg_steps': 0.625, 'epoch': 0.77}
|
|||
|
|
77%|███████████████████████████████████████████████████████████████████████████████████████▌ | 523/681 [34:04<06:36, 2.51s/it]
77%|███████████████████████████████████████████████████████████████████████████████████████▋ | 524/681 [34:07<06:45, 2.58s/it]
{'loss': 0.8791, 'grad_norm': 34.071510314941406, 'learning_rate': 7.781828926091535e-08, 'rewards/chosen': -0.296951562166214, 'rewards/rejected': -0.9517253637313843, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.6547738313674927, 'logps/chosen': -233.17068481445312, 'logps/rejected': -683.9710083007812, 'logps/ref_chosen': -47.35930252075195, 'logps/ref_rejected': -87.55693054199219, 'logits/chosen': -3.0543670654296875, 'logits/rejected': -3.656444549560547, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.001596899121068418, 'epsilon_dpo/loss_margin_mean': 410.6026916503906, 'epsilon_dpo/beta_margin_mean': 0.6547738313674927, 'epsilon_dpo/beta_margin_std': 0.4421177804470062, 'epsilon_dpo/beta_margin_grad_mean': -0.3489263653755188, 'epsilon_dpo/beta_margin_grad_std': 0.09371962398290634, 'kl/beta': 0.0016108342679217458, 'kl/avg_steps': 0.875, 'epoch': 0.77}
|
|||
|
|
77%|███████████████████████████████████████████████████████████████████████████████████████▋ | 524/681 [34:07<06:45, 2.58s/it]
77%|███████████████████████████████████████████████████████████████████████████████████████▉ | 525/681 [34:09<06:34, 2.53s/it]
{'loss': 0.8517, 'grad_norm': 33.28862380981445, 'learning_rate': 7.689012058193384e-08, 'rewards/chosen': -0.27915799617767334, 'rewards/rejected': -1.0120179653167725, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.7328599691390991, 'logps/chosen': -205.41815185546875, 'logps/rejected': -747.1358642578125, 'logps/ref_chosen': -29.801021575927734, 'logps/ref_rejected': -107.73757934570312, 'logits/chosen': -2.9925098419189453, 'logits/rejected': -3.6460928916931152, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.001583546632900834, 'epsilon_dpo/loss_margin_mean': 463.7811584472656, 'epsilon_dpo/beta_margin_mean': 0.7328599691390991, 'epsilon_dpo/beta_margin_std': 0.5668560862541199, 'epsilon_dpo/beta_margin_grad_mean': -0.33597642183303833, 'epsilon_dpo/beta_margin_grad_std': 0.11459914594888687, 'kl/beta': 0.0015968617517501116, 'kl/avg_steps': 0.84375, 'epoch': 0.77}
|
|||
|
|
77%|███████████████████████████████████████████████████████████████████████████████████████▉ | 525/681 [34:09<06:34, 2.53s/it]
77%|████████████████████████████████████████████████████████████████████████████████████████ | 526/681 [34:12<06:30, 2.52s/it]
{'loss': 0.8619, 'grad_norm': 33.44818115234375, 'learning_rate': 7.596651350926836e-08, 'rewards/chosen': -0.2869272232055664, 'rewards/rejected': -0.963952898979187, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.6770256757736206, 'logps/chosen': -219.9847412109375, 'logps/rejected': -707.4378662109375, 'logps/ref_chosen': -37.456085205078125, 'logps/ref_rejected': -93.045654296875, 'logits/chosen': -2.96604061126709, 'logits/rejected': -3.718456983566284, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0015698021743446589, 'epsilon_dpo/loss_margin_mean': 431.86358642578125, 'epsilon_dpo/beta_margin_mean': 0.6770256757736206, 'epsilon_dpo/beta_margin_std': 0.4311753809452057, 'epsilon_dpo/beta_margin_grad_mean': -0.3435947597026825, 'epsilon_dpo/beta_margin_grad_std': 0.0916726216673851, 'kl/beta': 0.001583500881679356, 'kl/avg_steps': 0.875, 'epoch': 0.77}
|
|||
|
|
77%|████████████████████████████████████████████████████████████████████████████████████████ | 526/681 [34:12<06:30, 2.52s/it]
77%|████████████████████████████████████████████████████████████████████████████████████████▏ | 527/681 [34:14<06:28, 2.52s/it]
{'loss': 0.9152, 'grad_norm': 33.482730865478516, 'learning_rate': 7.504749238082414e-08, 'rewards/chosen': -0.27390962839126587, 'rewards/rejected': -0.8795421123504639, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.6056324243545532, 'logps/chosen': -219.21990966796875, 'logps/rejected': -649.338623046875, 'logps/ref_chosen': -43.55718231201172, 'logps/ref_rejected': -83.77217102050781, 'logits/chosen': -3.048184871673584, 'logits/rejected': -3.7029123306274414, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0015561856562271714, 'epsilon_dpo/loss_margin_mean': 389.9037170410156, 'epsilon_dpo/beta_margin_mean': 0.6056324243545532, 'epsilon_dpo/beta_margin_std': 0.44141340255737305, 'epsilon_dpo/beta_margin_grad_mean': -0.35842540860176086, 'epsilon_dpo/beta_margin_grad_std': 0.09837586432695389, 'kl/beta': 0.0015697655035182834, 'kl/avg_steps': 0.875, 'epoch': 0.77}
|
|||
|
|
77%|████████████████████████████████████████████████████████████████████████████████████████▏ | 527/681 [34:14<06:28, 2.52s/it]
78%|████████████████████████████████████████████████████████████████████████████████████████▍ | 528/681 [34:17<06:35, 2.58s/it]
{'loss': 0.9063, 'grad_norm': 36.37355041503906, 'learning_rate': 7.413308141366254e-08, 'rewards/chosen': -0.3164703845977783, 'rewards/rejected': -0.971566915512085, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.6550965309143066, 'logps/chosen': -250.51231384277344, 'logps/rejected': -730.7449951171875, 'logps/ref_chosen': -46.319541931152344, 'logps/ref_rejected': -101.05429077148438, 'logits/chosen': -3.052718162536621, 'logits/rejected': -3.7526984214782715, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0015451188664883375, 'epsilon_dpo/loss_margin_mean': 425.4979248046875, 'epsilon_dpo/beta_margin_mean': 0.6550965309143066, 'epsilon_dpo/beta_margin_std': 0.5716320276260376, 'epsilon_dpo/beta_margin_grad_mean': -0.35285684466362, 'epsilon_dpo/beta_margin_grad_std': 0.11853712797164917, 'kl/beta': 0.0015561492182314396, 'kl/avg_steps': 0.71875, 'epoch': 0.78}
|
|||
|
|
78%|████████████████████████████████████████████████████████████████████████████████████████▍ | 528/681 [34:17<06:35, 2.58s/it]
78%|████████████████████████████████████████████████████████████████████████████████████████▌ | 529/681 [34:19<06:28, 2.55s/it]
{'loss': 0.987, 'grad_norm': 36.469417572021484, 'learning_rate': 7.322330470336313e-08, 'rewards/chosen': -0.41968581080436707, 'rewards/rejected': -0.9510923624038696, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.531406581401825, 'logps/chosen': -311.4798889160156, 'logps/rejected': -714.68115234375, 'logps/ref_chosen': -39.26038360595703, 'logps/ref_rejected': -94.54532623291016, 'logits/chosen': -3.203193187713623, 'logits/rejected': -3.8966808319091797, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0015350582543760538, 'epsilon_dpo/loss_margin_mean': 347.9163513183594, 'epsilon_dpo/beta_margin_mean': 0.531406581401825, 'epsilon_dpo/beta_margin_std': 0.5209585428237915, 'epsilon_dpo/beta_margin_grad_mean': -0.3771320581436157, 'epsilon_dpo/beta_margin_grad_std': 0.11737023293972015, 'kl/beta': 0.0015450441278517246, 'kl/avg_steps': 0.65625, 'epoch': 0.78}
|
|||
|
|
78%|████████████████████████████████████████████████████████████████████████████████████████▌ | 529/681 [34:19<06:28, 2.55s/it]
78%|████████████████████████████████████████████████████████████████████████████████████████▋ | 530/681 [34:22<06:23, 2.54s/it]
{'loss': 0.9029, 'grad_norm': 38.925601959228516, 'learning_rate': 7.231818622338822e-08, 'rewards/chosen': -0.264974981546402, 'rewards/rejected': -0.9302804470062256, 'rewards/accuracies': 0.875, 'rewards/margins': 0.665305495262146, 'logps/chosen': -200.4312744140625, 'logps/rejected': -704.1270751953125, 'logps/ref_chosen': -27.232646942138672, 'logps/ref_rejected': -92.85646057128906, 'logits/chosen': -3.1173648834228516, 'logits/rejected': -3.8287353515625, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0015236110193654895, 'epsilon_dpo/loss_margin_mean': 438.0719299316406, 'epsilon_dpo/beta_margin_mean': 0.665305495262146, 'epsilon_dpo/beta_margin_std': 0.6024741530418396, 'epsilon_dpo/beta_margin_grad_mean': -0.3523128926753998, 'epsilon_dpo/beta_margin_grad_std': 0.11749440431594849, 'kl/beta': 0.0015349709428846836, 'kl/avg_steps': 0.75, 'epoch': 0.78}
|
|||
|
|
78%|████████████████████████████████████████████████████████████████████████████████████████▋ | 530/681 [34:22<06:23, 2.54s/it]
78%|████████████████████████████████████████████████████████████████████████████████████████▉ | 531/681 [34:24<06:12, 2.48s/it]
{'loss': 0.9395, 'grad_norm': 39.48409652709961, 'learning_rate': 7.141774982445147e-08, 'rewards/chosen': -0.2982354760169983, 'rewards/rejected': -0.8788637518882751, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.5806282758712769, 'logps/chosen': -226.21939086914062, 'logps/rejected': -659.6124267578125, 'logps/ref_chosen': -29.433807373046875, 'logps/ref_rejected': -77.6962890625, 'logits/chosen': -3.1506361961364746, 'logits/rejected': -3.9508585929870605, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0015113166300579906, 'epsilon_dpo/loss_margin_mean': 385.13055419921875, 'epsilon_dpo/beta_margin_mean': 0.5806282758712769, 'epsilon_dpo/beta_margin_std': 0.4778251051902771, 'epsilon_dpo/beta_margin_grad_mean': -0.36579129099845886, 'epsilon_dpo/beta_margin_grad_std': 0.10368164628744125, 'kl/beta': 0.0015235443133860826, 'kl/avg_steps': 0.8125, 'epoch': 0.78}
|
|||
|
|
78%|████████████████████████████████████████████████████████████████████████████████████████▉ | 531/681 [34:24<06:12, 2.48s/it]
78%|█████████████████████████████████████████████████████████████████████████████████████████ | 532/681 [34:27<06:10, 2.49s/it]
{'loss': 0.9702, 'grad_norm': 41.38243103027344, 'learning_rate': 7.052201923388953e-08, 'rewards/chosen': -0.3671477138996124, 'rewards/rejected': -0.9502686858177185, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.5831209421157837, 'logps/chosen': -292.22418212890625, 'logps/rejected': -727.0375366210938, 'logps/ref_chosen': -49.116477966308594, 'logps/ref_rejected': -93.35752868652344, 'logits/chosen': -3.245467185974121, 'logits/rejected': -3.924285888671875, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0015024424064904451, 'epsilon_dpo/loss_margin_mean': 390.57232666015625, 'epsilon_dpo/beta_margin_mean': 0.5831210017204285, 'epsilon_dpo/beta_margin_std': 0.6058715581893921, 'epsilon_dpo/beta_margin_grad_mean': -0.36811915040016174, 'epsilon_dpo/beta_margin_grad_std': 0.13502071797847748, 'kl/beta': 0.0015112652909010649, 'kl/avg_steps': 0.59375, 'epoch': 0.78}
|
|||
|
|
78%|█████████████████████████████████████████████████████████████████████████████████████████ | 532/681 [34:27<06:10, 2.49s/it]
78%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 533/681 [34:29<06:08, 2.49s/it]
{'loss': 0.9883, 'grad_norm': 38.90617752075195, 'learning_rate': 6.963101805503646e-08, 'rewards/chosen': -0.35740402340888977, 'rewards/rejected': -0.8978468179702759, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.5404428243637085, 'logps/chosen': -277.5054016113281, 'logps/rejected': -684.0938110351562, 'logps/ref_chosen': -39.03050994873047, 'logps/ref_rejected': -81.61624145507812, 'logits/chosen': -3.1959118843078613, 'logits/rejected': -3.9355404376983643, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0014921656111255288, 'epsilon_dpo/loss_margin_mean': 364.0026550292969, 'epsilon_dpo/beta_margin_mean': 0.5404428243637085, 'epsilon_dpo/beta_margin_std': 0.5591951012611389, 'epsilon_dpo/beta_margin_grad_mean': -0.3761315643787384, 'epsilon_dpo/beta_margin_grad_std': 0.12172158807516098, 'kl/beta': 0.0015023450832813978, 'kl/avg_steps': 0.6875, 'epoch': 0.78}
|
|||
|
|
78%|█████████████████████████████████████████████████████████████████████████████████████████▏ | 533/681 [34:29<06:08, 2.49s/it]
78%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 534/681 [34:32<06:09, 2.51s/it]
{'loss': 0.8956, 'grad_norm': 43.0300178527832, 'learning_rate': 6.874476976660184e-08, 'rewards/chosen': -0.26844221353530884, 'rewards/rejected': -0.9091461896896362, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.6407039165496826, 'logps/chosen': -213.8875732421875, 'logps/rejected': -700.2742919921875, 'logps/ref_chosen': -33.198890686035156, 'logps/ref_rejected': -85.75593566894531, 'logits/chosen': -3.1139631271362305, 'logits/rejected': -3.9010872840881348, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0014812753070145845, 'epsilon_dpo/loss_margin_mean': 433.8296813964844, 'epsilon_dpo/beta_margin_mean': 0.6407039165496826, 'epsilon_dpo/beta_margin_std': 0.46845534443855286, 'epsilon_dpo/beta_margin_grad_mean': -0.35220903158187866, 'epsilon_dpo/beta_margin_grad_std': 0.10367898643016815, 'kl/beta': 0.0014920870307832956, 'kl/avg_steps': 0.734375, 'epoch': 0.78}
|
|||
|
|
78%|█████████████████████████████████████████████████████████████████████████████████████████▍ | 534/681 [34:32<06:09, 2.51s/it]
79%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 535/681 [34:35<06:14, 2.56s/it]
{'loss': 0.8681, 'grad_norm': 40.018455505371094, 'learning_rate': 6.786329772205246e-08, 'rewards/chosen': -0.3082220256328583, 'rewards/rejected': -1.0216362476348877, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.7134143114089966, 'logps/chosen': -248.54649353027344, 'logps/rejected': -791.900634765625, 'logps/ref_chosen': -39.410072326660156, 'logps/ref_rejected': -95.90589904785156, 'logits/chosen': -3.2459397315979004, 'logits/rejected': -4.044425010681152, 'kl/p_epsilon_steps': 0.953125, 'kl/n_epsilon_steps': 0.046875, 'epsilon_dpo/beta': 0.0014679327141493559, 'epsilon_dpo/loss_margin_mean': 486.8583068847656, 'epsilon_dpo/beta_margin_mean': 0.7134142518043518, 'epsilon_dpo/beta_margin_std': 0.5855581164360046, 'epsilon_dpo/beta_margin_grad_mean': -0.34026262164115906, 'epsilon_dpo/beta_margin_grad_std': 0.11611247062683105, 'kl/beta': 0.0014812094159424305, 'kl/avg_steps': 0.90625, 'epoch': 0.79}
|
|||
|
|
79%|█████████████████████████████████████████████████████████████████████████████████████████▌ | 535/681 [34:35<06:14, 2.56s/it]
79%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 536/681 [34:37<06:11, 2.56s/it]
{'loss': 0.8622, 'grad_norm': 43.79270553588867, 'learning_rate': 6.698662514899638e-08, 'rewards/chosen': -0.33760565519332886, 'rewards/rejected': -1.0515797138214111, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.7139739990234375, 'logps/chosen': -262.8885498046875, 'logps/rejected': -818.2322387695312, 'logps/ref_chosen': -31.54595184326172, 'logps/ref_rejected': -95.49956512451172, 'logits/chosen': -3.22434139251709, 'logits/rejected': -4.060914039611816, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.001455666613765061, 'epsilon_dpo/loss_margin_mean': 491.39007568359375, 'epsilon_dpo/beta_margin_mean': 0.7139739990234375, 'epsilon_dpo/beta_margin_std': 0.577499270439148, 'epsilon_dpo/beta_margin_grad_mean': -0.34118029475212097, 'epsilon_dpo/beta_margin_grad_std': 0.1087789535522461, 'kl/beta': 0.0014679065207019448, 'kl/avg_steps': 0.84375, 'epoch': 0.79}
|
|||
|
|
79%|█████████████████████████████████████████████████████████████████████████████████████████▋ | 536/681 [34:37<06:11, 2.56s/it]
79%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 537/681 [34:40<06:10, 2.57s/it]
{'loss': 0.9176, 'grad_norm': 37.07543182373047, 'learning_rate': 6.611477514857114e-08, 'rewards/chosen': -0.2549804449081421, 'rewards/rejected': -0.8682868480682373, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.6133064031600952, 'logps/chosen': -205.30322265625, 'logps/rejected': -680.2078857421875, 'logps/ref_chosen': -29.482723236083984, 'logps/ref_rejected': -78.68902587890625, 'logits/chosen': -3.1620330810546875, 'logits/rejected': -3.9779064655303955, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0014448519796133041, 'epsilon_dpo/loss_margin_mean': 425.6983947753906, 'epsilon_dpo/beta_margin_mean': 0.6133064031600952, 'epsilon_dpo/beta_margin_std': 0.48698103427886963, 'epsilon_dpo/beta_margin_grad_mean': -0.3592894971370697, 'epsilon_dpo/beta_margin_grad_std': 0.10450321435928345, 'kl/beta': 0.0014556247042492032, 'kl/avg_steps': 0.75, 'epoch': 0.79}
|
|||
|
|
79%|█████████████████████████████████████████████████████████████████████████████████████████▉ | 537/681 [34:40<06:10, 2.57s/it]
79%|██████████████████████████████████████████████████████████████████████████████████████████ | 538/681 [34:42<05:55, 2.49s/it]
{'loss': 0.8867, 'grad_norm': 43.92093276977539, 'learning_rate': 6.524777069483525e-08, 'rewards/chosen': -0.4264609217643738, 'rewards/rejected': -1.1388074159622192, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.7123464345932007, 'logps/chosen': -342.5325622558594, 'logps/rejected': -885.6788330078125, 'logps/ref_chosen': -46.485496520996094, 'logps/ref_rejected': -91.08615112304688, 'logits/chosen': -3.295565128326416, 'logits/rejected': -4.157257080078125, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0014349992852658033, 'epsilon_dpo/loss_margin_mean': 498.54559326171875, 'epsilon_dpo/beta_margin_mean': 0.7123464941978455, 'epsilon_dpo/beta_margin_std': 0.6496385931968689, 'epsilon_dpo/beta_margin_grad_mean': -0.34261569380760193, 'epsilon_dpo/beta_margin_grad_std': 0.132565438747406, 'kl/beta': 0.0014447887660935521, 'kl/avg_steps': 0.6875, 'epoch': 0.79}
|
|||
|
|
79%|██████████████████████████████████████████████████████████████████████████████████████████ | 538/681 [34:42<05:55, 2.49s/it]
79%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 539/681 [34:45<05:54, 2.50s/it]
{'loss': 0.8907, 'grad_norm': 45.51395034790039, 'learning_rate': 6.438563463416221e-08, 'rewards/chosen': -0.4068090319633484, 'rewards/rejected': -1.0954333543777466, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.688624382019043, 'logps/chosen': -323.3985595703125, 'logps/rejected': -864.95458984375, 'logps/ref_chosen': -39.210296630859375, 'logps/ref_rejected': -95.66792297363281, 'logits/chosen': -3.2940754890441895, 'logits/rejected': -4.077099800109863, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.001425649388693273, 'epsilon_dpo/loss_margin_mean': 485.098388671875, 'epsilon_dpo/beta_margin_mean': 0.6886243224143982, 'epsilon_dpo/beta_margin_std': 0.60282963514328, 'epsilon_dpo/beta_margin_grad_mean': -0.34682291746139526, 'epsilon_dpo/beta_margin_grad_std': 0.12398696690797806, 'kl/beta': 0.0014349236153066158, 'kl/avg_steps': 0.65625, 'epoch': 0.79}
|
|||
|
|
79%|██████████████████████████████████████████████████████████████████████████████████████████▏ | 539/681 [34:45<05:54, 2.50s/it]
79%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 540/681 [34:47<05:52, 2.50s/it]
{'loss': 0.8461, 'grad_norm': 49.0323486328125, 'learning_rate': 6.352838968463919e-08, 'rewards/chosen': -0.47576338052749634, 'rewards/rejected': -1.310936450958252, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.8351730108261108, 'logps/chosen': -385.478759765625, 'logps/rejected': -1051.9365234375, 'logps/ref_chosen': -50.91047668457031, 'logps/ref_rejected': -125.03810119628906, 'logits/chosen': -3.369414806365967, 'logits/rejected': -4.240825176239014, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0014163546729832888, 'epsilon_dpo/loss_margin_mean': 592.330078125, 'epsilon_dpo/beta_margin_mean': 0.8351730108261108, 'epsilon_dpo/beta_margin_std': 0.7946527004241943, 'epsilon_dpo/beta_margin_grad_mean': -0.32467931509017944, 'epsilon_dpo/beta_margin_grad_std': 0.1530696600675583, 'kl/beta': 0.0014255683636292815, 'kl/avg_steps': 0.65625, 'epoch': 0.79}
|
|||
|
|
79%|██████████████████████████████████████████████████████████████████████████████████████████▍ | 540/681 [34:47<05:52, 2.50s/it]
79%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 541/681 [34:50<05:53, 2.53s/it]
{'loss': 0.8668, 'grad_norm': 54.5290412902832, 'learning_rate': 6.267605843546767e-08, 'rewards/chosen': -0.4690607190132141, 'rewards/rejected': -1.2190579175949097, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.7499971985816956, 'logps/chosen': -380.36578369140625, 'logps/rejected': -977.6619873046875, 'logps/ref_chosen': -48.1887321472168, 'logps/ref_rejected': -110.07643127441406, 'logits/chosen': -3.3453049659729004, 'logits/rejected': -4.137938499450684, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0014062352711334825, 'epsilon_dpo/loss_margin_mean': 535.408447265625, 'epsilon_dpo/beta_margin_mean': 0.7499971985816956, 'epsilon_dpo/beta_margin_std': 0.6641453504562378, 'epsilon_dpo/beta_margin_grad_mean': -0.3354036808013916, 'epsilon_dpo/beta_margin_grad_std': 0.13668367266654968, 'kl/beta': 0.0014162741135805845, 'kl/avg_steps': 0.71875, 'epoch': 0.79}
|
|||
|
|
79%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 541/681 [34:50<05:53, 2.53s/it]
80%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 542/681 [34:52<05:53, 2.54s/it]
{'loss': 0.8216, 'grad_norm': 42.56067657470703, 'learning_rate': 6.182866334636888e-08, 'rewards/chosen': -0.3652591109275818, 'rewards/rejected': -1.2295722961425781, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.8643131852149963, 'logps/chosen': -294.858642578125, 'logps/rejected': -986.3819580078125, 'logps/ref_chosen': -34.308189392089844, 'logps/ref_rejected': -104.48388671875, 'logits/chosen': -3.2921829223632812, 'logits/rejected': -4.272438049316406, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0013966395054012537, 'epsilon_dpo/loss_margin_mean': 621.3475952148438, 'epsilon_dpo/beta_margin_mean': 0.8643131852149963, 'epsilon_dpo/beta_margin_std': 0.7869569063186646, 'epsilon_dpo/beta_margin_grad_mean': -0.31941473484039307, 'epsilon_dpo/beta_margin_grad_std': 0.1469242125749588, 'kl/beta': 0.0014061672845855355, 'kl/avg_steps': 0.6875, 'epoch': 0.8}
|
|||
|
|
80%|██████████████████████████████████████████████████████████████████████████████████████████▋ | 542/681 [34:52<05:53, 2.54s/it]
80%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 543/681 [34:55<05:49, 2.53s/it]
{'loss': 0.8507, 'grad_norm': 50.612884521484375, 'learning_rate': 6.098622674699147e-08, 'rewards/chosen': -0.527320921421051, 'rewards/rejected': -1.398918867111206, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.8715978860855103, 'logps/chosen': -422.712646484375, 'logps/rejected': -1121.7271728515625, 'logps/ref_chosen': -43.612937927246094, 'logps/ref_rejected': -111.56673431396484, 'logits/chosen': -3.523017168045044, 'logits/rejected': -4.250926971435547, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0013875396689400077, 'epsilon_dpo/loss_margin_mean': 631.060791015625, 'epsilon_dpo/beta_margin_mean': 0.8715978860855103, 'epsilon_dpo/beta_margin_std': 0.9173281192779541, 'epsilon_dpo/beta_margin_grad_mean': -0.32512134313583374, 'epsilon_dpo/beta_margin_grad_std': 0.160647913813591, 'kl/beta': 0.0013965658145025373, 'kl/avg_steps': 0.65625, 'epoch': 0.8}
|
|||
|
|
80%|██████████████████████████████████████████████████████████████████████████████████████████▉ | 543/681 [34:55<05:49, 2.53s/it]
80%|███████████████████████████████████████████████████████████████████████████████████████████ | 544/681 [34:57<05:47, 2.53s/it]
{'loss': 0.8257, 'grad_norm': 46.56395721435547, 'learning_rate': 6.01487708363232e-08, 'rewards/chosen': -0.5249900221824646, 'rewards/rejected': -1.464677095413208, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.9396871328353882, 'logps/chosen': -419.0536193847656, 'logps/rejected': -1170.40087890625, 'logps/ref_chosen': -39.47812271118164, 'logps/ref_rejected': -106.17308044433594, 'logits/chosen': -3.383917808532715, 'logits/rejected': -4.392078876495361, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.001378059620037675, 'epsilon_dpo/loss_margin_mean': 684.6524047851562, 'epsilon_dpo/beta_margin_mean': 0.9396871328353882, 'epsilon_dpo/beta_margin_std': 1.0003536939620972, 'epsilon_dpo/beta_margin_grad_mean': -0.3185425102710724, 'epsilon_dpo/beta_margin_grad_std': 0.15885071456432343, 'kl/beta': 0.0013874606229364872, 'kl/avg_steps': 0.6875, 'epoch': 0.8}
|
|||
|
|
80%|███████████████████████████████████████████████████████████████████████████████████████████ | 544/681 [34:57<05:47, 2.53s/it]
80%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 545/681 [35:00<05:48, 2.57s/it]
{'loss': 0.8128, 'grad_norm': 44.38985061645508, 'learning_rate': 5.9316317682106294e-08, 'rewards/chosen': -0.5297784805297852, 'rewards/rejected': -1.4392656087875366, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.9094871282577515, 'logps/chosen': -428.8343505859375, 'logps/rejected': -1154.3603515625, 'logps/ref_chosen': -42.28656768798828, 'logps/ref_rejected': -100.58802032470703, 'logits/chosen': -3.4539644718170166, 'logits/rejected': -4.4015960693359375, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.001366927521303296, 'epsilon_dpo/loss_margin_mean': 667.2245483398438, 'epsilon_dpo/beta_margin_mean': 0.9094871282577515, 'epsilon_dpo/beta_margin_std': 0.8563925623893738, 'epsilon_dpo/beta_margin_grad_mean': -0.3136303424835205, 'epsilon_dpo/beta_margin_grad_std': 0.1529344618320465, 'kl/beta': 0.001377986976876855, 'kl/avg_steps': 0.8125, 'epoch': 0.8}
|
|||
|
|
80%|███████████████████████████████████████████████████████████████████████████████████████████▏ | 545/681 [35:00<05:48, 2.57s/it]
80%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 546/681 [35:02<05:48, 2.58s/it]
{'loss': 0.8927, 'grad_norm': 47.50184631347656, 'learning_rate': 5.848888922025552e-08, 'rewards/chosen': -0.5232120156288147, 'rewards/rejected': -1.3013055324554443, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.7780935764312744, 'logps/chosen': -418.70166015625, 'logps/rejected': -1046.0960693359375, 'logps/ref_chosen': -35.013492584228516, 'logps/ref_rejected': -86.37733459472656, 'logits/chosen': -3.3195810317993164, 'logits/rejected': -4.246737957000732, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.0013584739062935114, 'epsilon_dpo/loss_margin_mean': 576.030517578125, 'epsilon_dpo/beta_margin_mean': 0.7780935764312744, 'epsilon_dpo/beta_margin_std': 0.8156005144119263, 'epsilon_dpo/beta_margin_grad_mean': -0.33587032556533813, 'epsilon_dpo/beta_margin_grad_std': 0.16217297315597534, 'kl/beta': 0.0013668810715898871, 'kl/avg_steps': 0.625, 'epoch': 0.8}
|
|||
|
|
80%|███████████████████████████████████████████████████████████████████████████████████████████▍ | 546/681 [35:03<05:48, 2.58s/it]
80%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 547/681 [35:05<05:44, 2.57s/it]
{'loss': 0.8106, 'grad_norm': 46.05626678466797, 'learning_rate': 5.7666507254280265e-08, 'rewards/chosen': -0.5293705463409424, 'rewards/rejected': -1.4406218528747559, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.9112513065338135, 'logps/chosen': -437.0877380371094, 'logps/rejected': -1164.623291015625, 'logps/ref_chosen': -45.810516357421875, 'logps/ref_rejected': -95.15408325195312, 'logits/chosen': -3.386657238006592, 'logits/rejected': -4.366371154785156, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0013491871068254113, 'epsilon_dpo/loss_margin_mean': 678.1919555664062, 'epsilon_dpo/beta_margin_mean': 0.9112512469291687, 'epsilon_dpo/beta_margin_std': 0.8325828909873962, 'epsilon_dpo/beta_margin_grad_mean': -0.3124694228172302, 'epsilon_dpo/beta_margin_grad_std': 0.15798072516918182, 'kl/beta': 0.0013583911349996924, 'kl/avg_steps': 0.6875, 'epoch': 0.8}
|
|||
|
|
80%|███████████████████████████████████████████████████████████████████████████████████████████▌ | 547/681 [35:05<05:44, 2.57s/it]
80%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 548/681 [35:08<05:43, 2.59s/it]
{'loss': 0.7973, 'grad_norm': 44.01782989501953, 'learning_rate': 5.684919345471029e-08, 'rewards/chosen': -0.4989817142486572, 'rewards/rejected': -1.4458205699920654, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.9468388557434082, 'logps/chosen': -411.8974914550781, 'logps/rejected': -1179.62451171875, 'logps/ref_chosen': -40.29045486450195, 'logps/ref_rejected': -98.83418273925781, 'logits/chosen': -3.3794431686401367, 'logits/rejected': -4.389235496520996, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0013399748131632805, 'epsilon_dpo/loss_margin_mean': 709.183349609375, 'epsilon_dpo/beta_margin_mean': 0.946838915348053, 'epsilon_dpo/beta_margin_std': 0.9356114864349365, 'epsilon_dpo/beta_margin_grad_mean': -0.3127719461917877, 'epsilon_dpo/beta_margin_grad_std': 0.14488740265369415, 'kl/beta': 0.001349115977063775, 'kl/avg_steps': 0.6875, 'epoch': 0.8}
|
|||
|
|
80%|███████████████████████████████████████████████████████████████████████████████████████████▋ | 548/681 [35:08<05:43, 2.59s/it]
81%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 549/681 [35:10<05:30, 2.51s/it]
{'loss': 0.9535, 'grad_norm': 81.65141296386719, 'learning_rate': 5.603696935852426e-08, 'rewards/chosen': -0.5441075563430786, 'rewards/rejected': -1.2614808082580566, 'rewards/accuracies': 0.828125, 'rewards/margins': 0.717373251914978, 'logps/chosen': -439.40911865234375, 'logps/rejected': -1030.0811767578125, 'logps/ref_chosen': -33.00281524658203, 'logps/ref_rejected': -81.72914123535156, 'logits/chosen': -3.2905972003936768, 'logits/rejected': -4.312599182128906, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0013320817379280925, 'epsilon_dpo/loss_margin_mean': 541.94580078125, 'epsilon_dpo/beta_margin_mean': 0.7173733115196228, 'epsilon_dpo/beta_margin_std': 0.8851867318153381, 'epsilon_dpo/beta_margin_grad_mean': -0.35267123579978943, 'epsilon_dpo/beta_margin_grad_std': 0.17075052857398987, 'kl/beta': 0.0013399041490629315, 'kl/avg_steps': 0.59375, 'epoch': 0.81}
|
|||
|
|
81%|███████████████████████████████████████████████████████████████████████████████████████████▉ | 549/681 [35:10<05:30, 2.51s/it]
81%|████████████████████████████████████████████████████████████████████████████████████████████ | 550/681 [35:13<05:31, 2.53s/it]
{'loss': 0.79, 'grad_norm': 47.79182052612305, 'learning_rate': 5.5229856368582376e-08, 'rewards/chosen': -0.5064237713813782, 'rewards/rejected': -1.4578301906585693, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.9514063596725464, 'logps/chosen': -415.656982421875, 'logps/rejected': -1205.10888671875, 'logps/ref_chosen': -33.51374053955078, 'logps/ref_rejected': -101.08897399902344, 'logits/chosen': -3.2790517807006836, 'logits/rejected': -4.2515106201171875, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0013221375411376357, 'epsilon_dpo/loss_margin_mean': 721.8765869140625, 'epsilon_dpo/beta_margin_mean': 0.9514063596725464, 'epsilon_dpo/beta_margin_std': 0.8637392520904541, 'epsilon_dpo/beta_margin_grad_mean': -0.30787116289138794, 'epsilon_dpo/beta_margin_grad_std': 0.15403428673744202, 'kl/beta': 0.001331995357759297, 'kl/avg_steps': 0.75, 'epoch': 0.81}
|
|||
|
|
81%|████████████████████████████████████████████████████████████████████████████████████████████ | 550/681 [35:13<05:31, 2.53s/it]
81%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 551/681 [35:15<05:25, 2.50s/it]
{'loss': 0.7468, 'grad_norm': 45.95981979370117, 'learning_rate': 5.4427875753062734e-08, 'rewards/chosen': -0.5882077217102051, 'rewards/rejected': -1.6824512481689453, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.0942435264587402, 'logps/chosen': -488.1016845703125, 'logps/rejected': -1400.064208984375, 'logps/ref_chosen': -42.42958068847656, 'logps/ref_rejected': -117.51499938964844, 'logits/chosen': -3.347069501876831, 'logits/rejected': -4.404940605163574, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0013135350309312344, 'epsilon_dpo/loss_margin_mean': 836.8770751953125, 'epsilon_dpo/beta_margin_mean': 1.0942435264587402, 'epsilon_dpo/beta_margin_std': 0.9498324394226074, 'epsilon_dpo/beta_margin_grad_mean': -0.28273874521255493, 'epsilon_dpo/beta_margin_grad_std': 0.17567141354084015, 'kl/beta': 0.0013220797991380095, 'kl/avg_steps': 0.65625, 'epoch': 0.81}
|
|||
|
|
81%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 551/681 [35:15<05:25, 2.50s/it]
81%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 552/681 [35:17<05:18, 2.47s/it]
{'loss': 0.7312, 'grad_norm': 40.460792541503906, 'learning_rate': 5.363104864490034e-08, 'rewards/chosen': -0.5959450006484985, 'rewards/rejected': -1.787198543548584, 'rewards/accuracies': 0.921875, 'rewards/margins': 1.1912535429000854, 'logps/chosen': -496.78289794921875, 'logps/rejected': -1485.19287109375, 'logps/ref_chosen': -40.86379623413086, 'logps/ref_rejected': -112.38937377929688, 'logits/chosen': -3.3478827476501465, 'logits/rejected': -4.524975776672363, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0013033291324973106, 'epsilon_dpo/loss_margin_mean': 916.8844604492188, 'epsilon_dpo/beta_margin_mean': 1.1912535429000854, 'epsilon_dpo/beta_margin_std': 1.1001869440078735, 'epsilon_dpo/beta_margin_grad_mean': -0.27695924043655396, 'epsilon_dpo/beta_margin_grad_std': 0.17922824621200562, 'kl/beta': 0.0013134601758792996, 'kl/avg_steps': 0.78125, 'epoch': 0.81}
|
|||
|
|
81%|████████████████████████████████████████████████████████████████████████████████████████████▍ | 552/681 [35:17<05:18, 2.47s/it]
81%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 553/681 [35:20<05:25, 2.54s/it]
{'loss': 0.9607, 'grad_norm': 88.0752182006836, 'learning_rate': 5.2839396041230415e-08, 'rewards/chosen': -0.6639860272407532, 'rewards/rejected': -1.4159413576126099, 'rewards/accuracies': 0.796875, 'rewards/margins': 0.7519553303718567, 'logps/chosen': -561.1559448242188, 'logps/rejected': -1197.488037109375, 'logps/ref_chosen': -51.35152816772461, 'logps/ref_rejected': -103.84159851074219, 'logits/chosen': -3.496969699859619, 'logits/rejected': -4.206221580505371, 'kl/p_epsilon_steps': 0.734375, 'kl/n_epsilon_steps': 0.265625, 'epsilon_dpo/beta': 0.001297299051657319, 'epsilon_dpo/loss_margin_mean': 583.842041015625, 'epsilon_dpo/beta_margin_mean': 0.7519553303718567, 'epsilon_dpo/beta_margin_std': 1.001431941986084, 'epsilon_dpo/beta_margin_grad_mean': -0.3533886671066284, 'epsilon_dpo/beta_margin_grad_std': 0.17754045128822327, 'kl/beta': 0.001303278375416994, 'kl/avg_steps': 0.46875, 'epoch': 0.81}
|
|||
|
|
81%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 553/681 [35:20<05:25, 2.54s/it]
81%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 554/681 [35:23<05:22, 2.54s/it]
{'loss': 0.8105, 'grad_norm': 48.403167724609375, 'learning_rate': 5.205293880283551e-08, 'rewards/chosen': -0.5585451126098633, 'rewards/rejected': -1.4798319339752197, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.9212867021560669, 'logps/chosen': -478.4771728515625, 'logps/rejected': -1244.5457763671875, 'logps/ref_chosen': -46.820640563964844, 'logps/ref_rejected': -94.6780014038086, 'logits/chosen': -3.321976661682129, 'logits/rejected': -4.445281982421875, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.001288408413529396, 'epsilon_dpo/loss_margin_mean': 718.2112426757812, 'epsilon_dpo/beta_margin_mean': 0.9212867021560669, 'epsilon_dpo/beta_margin_std': 0.8502605557441711, 'epsilon_dpo/beta_margin_grad_mean': -0.31087225675582886, 'epsilon_dpo/beta_margin_grad_std': 0.16114763915538788, 'kl/beta': 0.0012971977703273296, 'kl/avg_steps': 0.6875, 'epoch': 0.81}
|
|||
|
|
81%|████████████████████████████████████████████████████████████████████████████████████████████▋ | 554/681 [35:23<05:22, 2.54s/it]
81%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 555/681 [35:25<05:16, 2.52s/it]
{'loss': 0.7069, 'grad_norm': 45.80141830444336, 'learning_rate': 5.127169765359515e-08, 'rewards/chosen': -0.49974268674850464, 'rewards/rejected': -1.677809238433838, 'rewards/accuracies': 0.890625, 'rewards/margins': 1.1780664920806885, 'logps/chosen': -423.08660888671875, 'logps/rejected': -1429.6494140625, 'logps/ref_chosen': -33.03218460083008, 'logps/ref_rejected': -115.6031265258789, 'logits/chosen': -3.4040637016296387, 'logits/rejected': -4.443061828613281, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.001279208343476057, 'epsilon_dpo/loss_margin_mean': 923.9918212890625, 'epsilon_dpo/beta_margin_mean': 1.1780664920806885, 'epsilon_dpo/beta_margin_std': 1.0308868885040283, 'epsilon_dpo/beta_margin_grad_mean': -0.27689388394355774, 'epsilon_dpo/beta_margin_grad_std': 0.16369499266147614, 'kl/beta': 0.0012883404269814491, 'kl/avg_steps': 0.71875, 'epoch': 0.81}
|
|||
|
|
81%|████████████████████████████████████████████████████████████████████████████████████████████▉ | 555/681 [35:25<05:16, 2.52s/it]
82%|█████████████████████████████████████████████████████████████████████████████████████████████ | 556/681 [35:28<05:10, 2.49s/it]
{'loss': 0.8334, 'grad_norm': 50.429317474365234, 'learning_rate': 5.049569317994012e-08, 'rewards/chosen': -0.5285704135894775, 'rewards/rejected': -1.413649559020996, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.8850791454315186, 'logps/chosen': -447.8577575683594, 'logps/rejected': -1221.822998046875, 'logps/ref_chosen': -32.514793395996094, 'logps/ref_rejected': -106.99200439453125, 'logits/chosen': -3.259387493133545, 'logits/rejected': -4.264613151550293, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.001270079636014998, 'epsilon_dpo/loss_margin_mean': 699.4880981445312, 'epsilon_dpo/beta_margin_mean': 0.8850791454315186, 'epsilon_dpo/beta_margin_std': 0.8993016481399536, 'epsilon_dpo/beta_margin_grad_mean': -0.3221823573112488, 'epsilon_dpo/beta_margin_grad_std': 0.15197478234767914, 'kl/beta': 0.001279146526940167, 'kl/avg_steps': 0.71875, 'epoch': 0.82}
|
|||
|
|
82%|█████████████████████████████████████████████████████████████████████████████████████████████ | 556/681 [35:28<05:10, 2.49s/it]
82%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 557/681 [35:30<05:16, 2.55s/it]
{'loss': 0.8139, 'grad_norm': 49.268924713134766, 'learning_rate': 4.9724945830310144e-08, 'rewards/chosen': -0.6233251094818115, 'rewards/rejected': -1.581774353981018, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.9584492444992065, 'logps/chosen': -543.5952758789062, 'logps/rejected': -1372.490966796875, 'logps/ref_chosen': -51.166099548339844, 'logps/ref_rejected': -117.06163024902344, 'logits/chosen': -3.3845365047454834, 'logits/rejected': -4.363818645477295, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.00126141298096627, 'epsilon_dpo/loss_margin_mean': 763.000244140625, 'epsilon_dpo/beta_margin_mean': 0.9584492444992065, 'epsilon_dpo/beta_margin_std': 0.9464617371559143, 'epsilon_dpo/beta_margin_grad_mean': -0.3104838728904724, 'epsilon_dpo/beta_margin_grad_std': 0.16874933242797852, 'kl/beta': 0.0012700182851403952, 'kl/avg_steps': 0.6875, 'epoch': 0.82}
|
|||
|
|
82%|█████████████████████████████████████████████████████████████████████████████████████████████▏ | 557/681 [35:30<05:16, 2.55s/it]
82%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 558/681 [35:33<05:14, 2.56s/it]
{'loss': 0.6774, 'grad_norm': 45.054534912109375, 'learning_rate': 4.8959475914614554e-08, 'rewards/chosen': -0.4374838173389435, 'rewards/rejected': -1.6066837310791016, 'rewards/accuracies': 0.9375, 'rewards/margins': 1.16920006275177, 'logps/chosen': -387.129150390625, 'logps/rejected': -1394.381103515625, 'logps/ref_chosen': -38.345916748046875, 'logps/ref_rejected': -109.03158569335938, 'logits/chosen': -3.1819422245025635, 'logits/rejected': -4.413593292236328, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0012516174465417862, 'epsilon_dpo/loss_margin_mean': 936.5662841796875, 'epsilon_dpo/beta_margin_mean': 1.16920006275177, 'epsilon_dpo/beta_margin_std': 0.9076313972473145, 'epsilon_dpo/beta_margin_grad_mean': -0.27005571126937866, 'epsilon_dpo/beta_margin_grad_std': 0.14844605326652527, 'kl/beta': 0.0012613465078175068, 'kl/avg_steps': 0.78125, 'epoch': 0.82}
|
|||
|
|
82%|█████████████████████████████████████████████████████████████████████████████████████████████▍ | 558/681 [35:33<05:14, 2.56s/it]
82%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 559/681 [35:35<05:11, 2.55s/it]
{'loss': 0.8302, 'grad_norm': 63.11558151245117, 'learning_rate': 4.8199303603697614e-08, 'rewards/chosen': -0.6229937672615051, 'rewards/rejected': -1.5874824523925781, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.9644886255264282, 'logps/chosen': -542.97265625, 'logps/rejected': -1378.138427734375, 'logps/ref_chosen': -42.82691955566406, 'logps/ref_rejected': -98.61148071289062, 'logits/chosen': -3.2931509017944336, 'logits/rejected': -4.473822593688965, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0012430885108187795, 'epsilon_dpo/loss_margin_mean': 779.3812255859375, 'epsilon_dpo/beta_margin_mean': 0.9644886255264282, 'epsilon_dpo/beta_margin_std': 1.0081322193145752, 'epsilon_dpo/beta_margin_grad_mean': -0.31191176176071167, 'epsilon_dpo/beta_margin_grad_std': 0.17650018632411957, 'kl/beta': 0.0012515686685219407, 'kl/avg_steps': 0.6875, 'epoch': 0.82}
|
|||
|
|
82%|█████████████████████████████████████████████████████████████████████████████████████████████▌ | 559/681 [35:35<05:11, 2.55s/it]
82%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 560/681 [35:38<05:12, 2.58s/it]
{'loss': 0.9186, 'grad_norm': 46.32260513305664, 'learning_rate': 4.7444448928806615e-08, 'rewards/chosen': -0.5071742534637451, 'rewards/rejected': -1.2396838665008545, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.7325094938278198, 'logps/chosen': -445.64739990234375, 'logps/rejected': -1090.350341796875, 'logps/ref_chosen': -36.52899932861328, 'logps/ref_rejected': -85.29887390136719, 'logits/chosen': -3.276106834411621, 'logits/rejected': -4.303354740142822, 'kl/p_epsilon_steps': 0.765625, 'kl/n_epsilon_steps': 0.234375, 'epsilon_dpo/beta': 0.0012365429429337382, 'epsilon_dpo/loss_margin_mean': 595.9330444335938, 'epsilon_dpo/beta_margin_mean': 0.7325094938278198, 'epsilon_dpo/beta_margin_std': 0.8337624669075012, 'epsilon_dpo/beta_margin_grad_mean': -0.34858644008636475, 'epsilon_dpo/beta_margin_grad_std': 0.15379458665847778, 'kl/beta': 0.001243022852577269, 'kl/avg_steps': 0.53125, 'epoch': 0.82}
|
|||
|
|
82%|█████████████████████████████████████████████████████████████████████████████████████████████▋ | 560/681 [35:38<05:12, 2.58s/it]
82%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 561/681 [35:41<05:10, 2.59s/it]
{'loss': 0.8751, 'grad_norm': 51.50674057006836, 'learning_rate': 4.669493178106432e-08, 'rewards/chosen': -0.45402687788009644, 'rewards/rejected': -1.293859601020813, 'rewards/accuracies': 0.875, 'rewards/margins': 0.8398326635360718, 'logps/chosen': -401.35638427734375, 'logps/rejected': -1159.802490234375, 'logps/ref_chosen': -33.39714050292969, 'logps/ref_rejected': -104.86662292480469, 'logits/chosen': -3.1779091358184814, 'logits/rejected': -4.242127418518066, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.0012296221684664488, 'epsilon_dpo/loss_margin_mean': 686.9766845703125, 'epsilon_dpo/beta_margin_mean': 0.8398326635360718, 'epsilon_dpo/beta_margin_std': 0.8977148532867432, 'epsilon_dpo/beta_margin_grad_mean': -0.32985633611679077, 'epsilon_dpo/beta_margin_grad_std': 0.16898517310619354, 'kl/beta': 0.001236454234458506, 'kl/avg_steps': 0.5625, 'epoch': 0.82}
|
|||
|
|
82%|█████████████████████████████████████████████████████████████████████████████████████████████▉ | 561/681 [35:41<05:10, 2.59s/it]
83%|██████████████████████████████████████████████████████████████████████████████████████████████ | 562/681 [35:43<05:10, 2.61s/it]
{'loss': 0.8169, 'grad_norm': 58.34850311279297, 'learning_rate': 4.5950771910944596e-08, 'rewards/chosen': -0.4571970999240875, 'rewards/rejected': -1.363953948020935, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.9067568182945251, 'logps/chosen': -410.8808898925781, 'logps/rejected': -1224.1141357421875, 'logps/ref_chosen': -37.05860900878906, 'logps/ref_rejected': -104.74354553222656, 'logits/chosen': -3.1661272048950195, 'logits/rejected': -4.288407325744629, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.001219669939018786, 'epsilon_dpo/loss_margin_mean': 745.5482788085938, 'epsilon_dpo/beta_margin_mean': 0.9067568182945251, 'epsilon_dpo/beta_margin_std': 0.8777819275856018, 'epsilon_dpo/beta_margin_grad_mean': -0.31512200832366943, 'epsilon_dpo/beta_margin_grad_std': 0.15188972651958466, 'kl/beta': 0.001229538000188768, 'kl/avg_steps': 0.8125, 'epoch': 0.83}
|
|||
|
|
83%|██████████████████████████████████████████████████████████████████████████████████████████████ | 562/681 [35:43<05:10, 2.61s/it]
83%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 563/681 [35:46<05:05, 2.59s/it]
{'loss': 0.8805, 'grad_norm': 56.34610366821289, 'learning_rate': 4.521198892775202e-08, 'rewards/chosen': -0.5091303586959839, 'rewards/rejected': -1.2900173664093018, 'rewards/accuracies': 0.875, 'rewards/margins': 0.7808871269226074, 'logps/chosen': -462.99176025390625, 'logps/rejected': -1168.041259765625, 'logps/ref_chosen': -44.557411193847656, 'logps/ref_rejected': -102.11149597167969, 'logits/chosen': -3.220041275024414, 'logits/rejected': -4.239965915679932, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0012125081848353148, 'epsilon_dpo/loss_margin_mean': 647.4954223632812, 'epsilon_dpo/beta_margin_mean': 0.7808871269226074, 'epsilon_dpo/beta_margin_std': 0.7981037497520447, 'epsilon_dpo/beta_margin_grad_mean': -0.33571353554725647, 'epsilon_dpo/beta_margin_grad_std': 0.15231700241565704, 'kl/beta': 0.0012196284951642156, 'kl/avg_steps': 0.59375, 'epoch': 0.83}
|
|||
|
|
83%|██████████████████████████████████████████████████████████████████████████████████████████████▏ | 563/681 [35:46<05:05, 2.59s/it]
83%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 564/681 [35:48<05:00, 2.57s/it]
{'loss': 0.7846, 'grad_norm': 49.93864059448242, 'learning_rate': 4.447860229910544e-08, 'rewards/chosen': -0.4852016270160675, 'rewards/rejected': -1.4298651218414307, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.944663405418396, 'logps/chosen': -445.9974060058594, 'logps/rejected': -1288.80322265625, 'logps/ref_chosen': -43.770145416259766, 'logps/ref_rejected': -99.07379150390625, 'logits/chosen': -3.3658523559570312, 'logits/rejected': -4.48719596862793, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0012034567771479487, 'epsilon_dpo/loss_margin_mean': 787.502197265625, 'epsilon_dpo/beta_margin_mean': 0.944663405418396, 'epsilon_dpo/beta_margin_std': 0.820910632610321, 'epsilon_dpo/beta_margin_grad_mean': -0.3051425516605377, 'epsilon_dpo/beta_margin_grad_std': 0.15122495591640472, 'kl/beta': 0.0012124297209084034, 'kl/avg_steps': 0.75, 'epoch': 0.83}
|
|||
|
|
83%|██████████████████████████████████████████████████████████████████████████████████████████████▍ | 564/681 [35:48<05:00, 2.57s/it]
83%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 565/681 [35:51<04:50, 2.51s/it]
{'loss': 0.8657, 'grad_norm': 44.75363540649414, 'learning_rate': 4.375063135042445e-08, 'rewards/chosen': -0.4435133934020996, 'rewards/rejected': -1.3005576133728027, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.8570442199707031, 'logps/chosen': -405.4493103027344, 'logps/rejected': -1182.4266357421875, 'logps/ref_chosen': -35.76008605957031, 'logps/ref_rejected': -92.45127868652344, 'logits/chosen': -3.1451213359832764, 'logits/rejected': -4.4003705978393555, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.001195436459966004, 'epsilon_dpo/loss_margin_mean': 720.2861328125, 'epsilon_dpo/beta_margin_mean': 0.8570442795753479, 'epsilon_dpo/beta_margin_std': 0.9114473462104797, 'epsilon_dpo/beta_margin_grad_mean': -0.3250107765197754, 'epsilon_dpo/beta_margin_grad_std': 0.1642923802137375, 'kl/beta': 0.001203404157422483, 'kl/avg_steps': 0.671875, 'epoch': 0.83}
|
|||
|
|
83%|██████████████████████████████████████████████████████████████████████████████████████████████▌ | 565/681 [35:51<04:50, 2.51s/it]
83%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 566/681 [35:53<04:50, 2.53s/it]
{'loss': 0.8029, 'grad_norm': 53.6401481628418, 'learning_rate': 4.3028095264420525e-08, 'rewards/chosen': -0.4532804787158966, 'rewards/rejected': -1.394671082496643, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.9413906335830688, 'logps/chosen': -425.1823425292969, 'logps/rejected': -1287.763427734375, 'logps/ref_chosen': -44.17033004760742, 'logps/ref_rejected': -111.02940368652344, 'logits/chosen': -3.165487289428711, 'logits/rejected': -4.408507347106934, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0011870847083628178, 'epsilon_dpo/loss_margin_mean': 795.7219848632812, 'epsilon_dpo/beta_margin_mean': 0.9413906335830688, 'epsilon_dpo/beta_margin_std': 0.8944268226623535, 'epsilon_dpo/beta_margin_grad_mean': -0.3126325309276581, 'epsilon_dpo/beta_margin_grad_std': 0.1551150530576706, 'kl/beta': 0.0011953727807849646, 'kl/avg_steps': 0.703125, 'epoch': 0.83}
|
|||
|
|
83%|██████████████████████████████████████████████████████████████████████████████████████████████▋ | 566/681 [35:53<04:50, 2.53s/it]
83%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 567/681 [35:56<04:41, 2.47s/it]
{'loss': 0.8671, 'grad_norm': 55.0069694519043, 'learning_rate': 4.231101308059165e-08, 'rewards/chosen': -0.4556201100349426, 'rewards/rejected': -1.2872684001922607, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.8316484093666077, 'logps/chosen': -419.98046875, 'logps/rejected': -1186.04541015625, 'logps/ref_chosen': -34.52465057373047, 'logps/ref_rejected': -92.00111389160156, 'logits/chosen': -3.2106685638427734, 'logits/rejected': -4.49782657623291, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.001178241684101522, 'epsilon_dpo/loss_margin_mean': 708.5885009765625, 'epsilon_dpo/beta_margin_mean': 0.8316484093666077, 'epsilon_dpo/beta_margin_std': 0.862678050994873, 'epsilon_dpo/beta_margin_grad_mean': -0.32961148023605347, 'epsilon_dpo/beta_margin_grad_std': 0.1618032306432724, 'kl/beta': 0.0011870265007019043, 'kl/avg_steps': 0.75, 'epoch': 0.83}
|
|||
|
|
83%|██████████████████████████████████████████████████████████████████████████████████████████████▉ | 567/681 [35:56<04:41, 2.47s/it]
83%|███████████████████████████████████████████████████████████████████████████████████████████████ | 568/681 [35:58<04:34, 2.43s/it]
{'loss': 0.7867, 'grad_norm': 51.46321487426758, 'learning_rate': 4.1599403694720145e-08, 'rewards/chosen': -0.37704116106033325, 'rewards/rejected': -1.316697359085083, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.9396562576293945, 'logps/chosen': -342.3359375, 'logps/rejected': -1224.04150390625, 'logps/ref_chosen': -20.8466854095459, 'logps/ref_rejected': -96.66067504882812, 'logits/chosen': -3.223677158355713, 'logits/rejected': -4.477365493774414, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0011691023828461766, 'epsilon_dpo/loss_margin_mean': 805.8914794921875, 'epsilon_dpo/beta_margin_mean': 0.9396561980247498, 'epsilon_dpo/beta_margin_std': 0.8285186886787415, 'epsilon_dpo/beta_margin_grad_mean': -0.3085906505584717, 'epsilon_dpo/beta_margin_grad_std': 0.14849402010440826, 'kl/beta': 0.0011781901121139526, 'kl/avg_steps': 0.78125, 'epoch': 0.83}
|
|||
|
|
83%|███████████████████████████████████████████████████████████████████████████████████████████████ | 568/681 [35:58<04:34, 2.43s/it]
84%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 569/681 [36:01<04:42, 2.52s/it]
{'loss': 0.8626, 'grad_norm': 49.56915283203125, 'learning_rate': 4.089328585837512e-08, 'rewards/chosen': -0.41802793741226196, 'rewards/rejected': -1.2753137350082397, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.857285737991333, 'logps/chosen': -403.1963195800781, 'logps/rejected': -1184.5604248046875, 'logps/ref_chosen': -44.985755920410156, 'logps/ref_rejected': -84.71590423583984, 'logits/chosen': -3.329655647277832, 'logits/rejected': -4.526782035827637, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0011607703054323792, 'epsilon_dpo/loss_margin_mean': 741.6339111328125, 'epsilon_dpo/beta_margin_mean': 0.8572857975959778, 'epsilon_dpo/beta_margin_std': 0.9337847232818604, 'epsilon_dpo/beta_margin_grad_mean': -0.3284705877304077, 'epsilon_dpo/beta_margin_grad_std': 0.159523144364357, 'kl/beta': 0.0011690568644553423, 'kl/avg_steps': 0.71875, 'epoch': 0.84}
|
|||
|
|
84%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 569/681 [36:01<04:42, 2.52s/it]
84%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 570/681 [36:03<04:39, 2.52s/it]
{'loss': 0.8506, 'grad_norm': 46.32669448852539, 'learning_rate': 4.019267817841834e-08, 'rewards/chosen': -0.5037564635276794, 'rewards/rejected': -1.3489649295806885, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.8452085256576538, 'logps/chosen': -474.98382568359375, 'logps/rejected': -1260.195068359375, 'logps/ref_chosen': -39.338470458984375, 'logps/ref_rejected': -88.34091186523438, 'logits/chosen': -3.3636648654937744, 'logits/rejected': -4.605798721313477, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0011524868896231055, 'epsilon_dpo/loss_margin_mean': 736.2088012695312, 'epsilon_dpo/beta_margin_mean': 0.8452085256576538, 'epsilon_dpo/beta_margin_std': 0.8515332341194153, 'epsilon_dpo/beta_margin_grad_mean': -0.3267635405063629, 'epsilon_dpo/beta_margin_grad_std': 0.1530372053384781, 'kl/beta': 0.0011607141932472587, 'kl/avg_steps': 0.71875, 'epoch': 0.84}
|
|||
|
|
84%|███████████████████████████████████████████████████████████████████████████████████████████████▍ | 570/681 [36:03<04:39, 2.52s/it]
84%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 571/681 [36:06<04:34, 2.50s/it]
{'loss': 0.7563, 'grad_norm': 43.56798553466797, 'learning_rate': 3.9497599116513705e-08, 'rewards/chosen': -0.4253048002719879, 'rewards/rejected': -1.4718279838562012, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.0465232133865356, 'logps/chosen': -398.0816650390625, 'logps/rejected': -1388.0191650390625, 'logps/ref_chosen': -27.42264747619629, 'logps/ref_rejected': -99.594482421875, 'logits/chosen': -3.367818832397461, 'logits/rejected': -4.655285835266113, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.001143542118370533, 'epsilon_dpo/loss_margin_mean': 917.7656860351562, 'epsilon_dpo/beta_margin_mean': 1.0465232133865356, 'epsilon_dpo/beta_margin_std': 0.9296081066131592, 'epsilon_dpo/beta_margin_grad_mean': -0.2934305667877197, 'epsilon_dpo/beta_margin_grad_std': 0.16104663908481598, 'kl/beta': 0.0011524311266839504, 'kl/avg_steps': 0.78125, 'epoch': 0.84}
|
|||
|
|
84%|███████████████████████████████████████████████████████████████████████████████████████████████▌ | 571/681 [36:06<04:34, 2.50s/it]
84%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 572/681 [36:08<04:32, 2.50s/it]
{'loss': 0.8689, 'grad_norm': 59.63137435913086, 'learning_rate': 3.880806698864086e-08, 'rewards/chosen': -0.4676932096481323, 'rewards/rejected': -1.3605258464813232, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.8928326368331909, 'logps/chosen': -437.66815185546875, 'logps/rejected': -1290.10791015625, 'logps/ref_chosen': -28.469844818115234, 'logps/ref_rejected': -90.8378677368164, 'logits/chosen': -3.412736654281616, 'logits/rejected': -4.5799455642700195, 'kl/p_epsilon_steps': 0.765625, 'kl/n_epsilon_steps': 0.234375, 'epsilon_dpo/beta': 0.001137536484748125, 'epsilon_dpo/loss_margin_mean': 790.0717163085938, 'epsilon_dpo/beta_margin_mean': 0.8928326964378357, 'epsilon_dpo/beta_margin_std': 0.9588112831115723, 'epsilon_dpo/beta_margin_grad_mean': -0.32172301411628723, 'epsilon_dpo/beta_margin_grad_std': 0.18587327003479004, 'kl/beta': 0.0011434975313022733, 'kl/avg_steps': 0.53125, 'epoch': 0.84}
|
|||
|
|
84%|███████████████████████████████████████████████████████████████████████████████████████████████▊ | 572/681 [36:08<04:32, 2.50s/it]
84%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 573/681 [36:11<04:28, 2.48s/it]
{'loss': 0.8899, 'grad_norm': 58.05121612548828, 'learning_rate': 3.812409996461275e-08, 'rewards/chosen': -0.5845237970352173, 'rewards/rejected': -1.4087841510772705, 'rewards/accuracies': 0.796875, 'rewards/margins': 0.8242603540420532, 'logps/chosen': -550.3379516601562, 'logps/rejected': -1339.390625, 'logps/ref_chosen': -35.605384826660156, 'logps/ref_rejected': -91.72334289550781, 'logits/chosen': -3.55238676071167, 'logits/rejected': -4.599684715270996, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.00113116973079741, 'epsilon_dpo/loss_margin_mean': 732.9346923828125, 'epsilon_dpo/beta_margin_mean': 0.8242603540420532, 'epsilon_dpo/beta_margin_std': 0.914152204990387, 'epsilon_dpo/beta_margin_grad_mean': -0.3352082371711731, 'epsilon_dpo/beta_margin_grad_std': 0.1706962138414383, 'kl/beta': 0.0011374547611922026, 'kl/avg_steps': 0.5625, 'epoch': 0.84}
|
|||
|
|
84%|███████████████████████████████████████████████████████████████████████████████████████████████▉ | 573/681 [36:11<04:28, 2.48s/it]
84%|████████████████████████████████████████████████████████████████████████████████████████████████ | 574/681 [36:13<04:29, 2.52s/it]
{'loss': 0.7582, 'grad_norm': 47.21782302856445, 'learning_rate': 3.74457160675965e-08, 'rewards/chosen': -0.4360452890396118, 'rewards/rejected': -1.4398435354232788, 'rewards/accuracies': 0.90625, 'rewards/margins': 1.003798246383667, 'logps/chosen': -419.362060546875, 'logps/rejected': -1383.410400390625, 'logps/ref_chosen': -31.43255615234375, 'logps/ref_rejected': -98.70758056640625, 'logits/chosen': -3.418008327484131, 'logits/rejected': -4.645315170288086, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0011223680339753628, 'epsilon_dpo/loss_margin_mean': 896.7732543945312, 'epsilon_dpo/beta_margin_mean': 1.003798246383667, 'epsilon_dpo/beta_margin_std': 0.8582514524459839, 'epsilon_dpo/beta_margin_grad_mean': -0.2976383864879608, 'epsilon_dpo/beta_margin_grad_std': 0.15246038138866425, 'kl/beta': 0.001131092431023717, 'kl/avg_steps': 0.78125, 'epoch': 0.84}
|
|||
|
|
84%|████████████████████████████████████████████████████████████████████████████████████████████████ | 574/681 [36:13<04:29, 2.52s/it]
84%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 575/681 [36:16<04:31, 2.57s/it]
{'loss': 0.8223, 'grad_norm': 52.89796447753906, 'learning_rate': 3.677293317363864e-08, 'rewards/chosen': -0.4872722625732422, 'rewards/rejected': -1.4568467140197754, 'rewards/accuracies': 0.875, 'rewards/margins': 0.9695743322372437, 'logps/chosen': -476.3949279785156, 'logps/rejected': -1411.6007080078125, 'logps/ref_chosen': -40.592437744140625, 'logps/ref_rejected': -102.49923706054688, 'logits/chosen': -3.360232353210449, 'logits/rejected': -4.744388580322266, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0011150705395266414, 'epsilon_dpo/loss_margin_mean': 873.2989501953125, 'epsilon_dpo/beta_margin_mean': 0.9695743918418884, 'epsilon_dpo/beta_margin_std': 1.0260599851608276, 'epsilon_dpo/beta_margin_grad_mean': -0.3144918978214264, 'epsilon_dpo/beta_margin_grad_std': 0.16865824162960052, 'kl/beta': 0.0011223242618143559, 'kl/avg_steps': 0.65625, 'epoch': 0.84}
|
|||
|
|
84%|████████████████████████████████████████████████████████████████████████████████████████████████▎ | 575/681 [36:16<04:31, 2.57s/it]
85%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 576/681 [36:18<04:24, 2.52s/it]
{'loss': 0.8987, 'grad_norm': 49.19211196899414, 'learning_rate': 3.6105769011194224e-08, 'rewards/chosen': -0.5297691822052002, 'rewards/rejected': -1.430067777633667, 'rewards/accuracies': 0.796875, 'rewards/margins': 0.9002985954284668, 'logps/chosen': -509.7559814453125, 'logps/rejected': -1400.2540283203125, 'logps/ref_chosen': -34.52962875366211, 'logps/ref_rejected': -107.82730102539062, 'logits/chosen': -3.442026376724243, 'logits/rejected': -4.685531139373779, 'kl/p_epsilon_steps': 0.765625, 'kl/n_epsilon_steps': 0.234375, 'epsilon_dpo/beta': 0.001109194359742105, 'epsilon_dpo/loss_margin_mean': 817.2003784179688, 'epsilon_dpo/beta_margin_mean': 0.9002985954284668, 'epsilon_dpo/beta_margin_std': 1.0915855169296265, 'epsilon_dpo/beta_margin_grad_mean': -0.3279215395450592, 'epsilon_dpo/beta_margin_grad_std': 0.1897161304950714, 'kl/beta': 0.0011150069767609239, 'kl/avg_steps': 0.53125, 'epoch': 0.85}
|
|||
|
|
85%|████████████████████████████████████████████████████████████████████████████████████████████████▍ | 576/681 [36:18<04:24, 2.52s/it]
85%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 577/681 [36:21<04:17, 2.47s/it]
{'loss': 0.8153, 'grad_norm': 47.395843505859375, 'learning_rate': 3.5444241160659304e-08, 'rewards/chosen': -0.4818669557571411, 'rewards/rejected': -1.3326964378356934, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.8508294224739075, 'logps/chosen': -472.676513671875, 'logps/rejected': -1302.816162109375, 'logps/ref_chosen': -36.379669189453125, 'logps/ref_rejected': -91.19670104980469, 'logits/chosen': -3.390251636505127, 'logits/rejected': -4.721263885498047, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0011009065201506019, 'epsilon_dpo/loss_margin_mean': 775.322509765625, 'epsilon_dpo/beta_margin_mean': 0.8508294224739075, 'epsilon_dpo/beta_margin_std': 0.7307493686676025, 'epsilon_dpo/beta_margin_grad_mean': -0.3192014694213867, 'epsilon_dpo/beta_margin_grad_std': 0.13799449801445007, 'kl/beta': 0.0011091148480772972, 'kl/avg_steps': 0.75, 'epoch': 0.85}
|
|||
|
|
85%|████████████████████████████████████████████████████████████████████████████████████████████████▌ | 577/681 [36:21<04:17, 2.47s/it]
85%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 578/681 [36:23<04:13, 2.46s/it]
{'loss': 0.8456, 'grad_norm': 56.689117431640625, 'learning_rate': 3.478836705390808e-08, 'rewards/chosen': -0.4083082675933838, 'rewards/rejected': -1.283423900604248, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.8751156330108643, 'logps/chosen': -402.2279357910156, 'logps/rejected': -1265.176513671875, 'logps/ref_chosen': -30.31075668334961, 'logps/ref_rejected': -89.46212005615234, 'logits/chosen': -3.35703182220459, 'logits/rejected': -4.623649597167969, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0010930553544312716, 'epsilon_dpo/loss_margin_mean': 803.7972412109375, 'epsilon_dpo/beta_margin_mean': 0.8751156330108643, 'epsilon_dpo/beta_margin_std': 0.8696184754371643, 'epsilon_dpo/beta_margin_grad_mean': -0.31814831495285034, 'epsilon_dpo/beta_margin_grad_std': 0.1622655838727951, 'kl/beta': 0.0011008584406226873, 'kl/avg_steps': 0.71875, 'epoch': 0.85}
|
|||
|
|
85%|████████████████████████████████████████████████████████████████████████████████████████████████▊ | 578/681 [36:23<04:13, 2.46s/it]
85%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 579/681 [36:26<04:12, 2.47s/it]
{'loss': 0.8296, 'grad_norm': 47.10636520385742, 'learning_rate': 3.41381639738331e-08, 'rewards/chosen': -0.48166942596435547, 'rewards/rejected': -1.3006623983383179, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.8189929723739624, 'logps/chosen': -481.96820068359375, 'logps/rejected': -1302.7352294921875, 'logps/ref_chosen': -38.872169494628906, 'logps/ref_rejected': -102.441162109375, 'logits/chosen': -3.4386825561523438, 'logits/rejected': -4.622516632080078, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0010845718206837773, 'epsilon_dpo/loss_margin_mean': 757.1980590820312, 'epsilon_dpo/beta_margin_mean': 0.8189929723739624, 'epsilon_dpo/beta_margin_std': 0.7138299345970154, 'epsilon_dpo/beta_margin_grad_mean': -0.3247964680194855, 'epsilon_dpo/beta_margin_grad_std': 0.13387812674045563, 'kl/beta': 0.0010930025018751621, 'kl/avg_steps': 0.78125, 'epoch': 0.85}
|
|||
|
|
85%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 579/681 [36:26<04:12, 2.47s/it]
85%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 580/681 [36:28<04:06, 2.45s/it]
{'loss': 0.9567, 'grad_norm': 53.16489028930664, 'learning_rate': 3.349364905389032e-08, 'rewards/chosen': -0.5118281841278076, 'rewards/rejected': -1.2188990116119385, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.7070707082748413, 'logps/chosen': -516.2334594726562, 'logps/rejected': -1220.0089111328125, 'logps/ref_chosen': -43.879005432128906, 'logps/ref_rejected': -87.93156433105469, 'logits/chosen': -3.4820971488952637, 'logits/rejected': -4.580034255981445, 'kl/p_epsilon_steps': 0.734375, 'kl/n_epsilon_steps': 0.265625, 'epsilon_dpo/beta': 0.0010795537382364273, 'epsilon_dpo/loss_margin_mean': 659.7229614257812, 'epsilon_dpo/beta_margin_mean': 0.7070707082748413, 'epsilon_dpo/beta_margin_std': 0.8788829445838928, 'epsilon_dpo/beta_margin_grad_mean': -0.3553590476512909, 'epsilon_dpo/beta_margin_grad_std': 0.1680212914943695, 'kl/beta': 0.0010845295619219542, 'kl/avg_steps': 0.46875, 'epoch': 0.85}
|
|||
|
|
85%|█████████████████████████████████████████████████████████████████████████████████████████████████ | 580/681 [36:28<04:06, 2.45s/it]
85%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 581/681 [36:30<04:02, 2.42s/it]
{'loss': 0.8701, 'grad_norm': 48.09032440185547, 'learning_rate': 3.285483927764726e-08, 'rewards/chosen': -0.5371580123901367, 'rewards/rejected': -1.3853325843811035, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.8481745719909668, 'logps/chosen': -542.8037109375, 'logps/rejected': -1391.629150390625, 'logps/ref_chosen': -43.27698516845703, 'logps/ref_rejected': -97.41041564941406, 'logits/chosen': -3.527688503265381, 'logits/rejected': -4.8321614265441895, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0010721554281190038, 'epsilon_dpo/loss_margin_mean': 794.6920166015625, 'epsilon_dpo/beta_margin_mean': 0.8481745719909668, 'epsilon_dpo/beta_margin_std': 0.9297420382499695, 'epsilon_dpo/beta_margin_grad_mean': -0.32997187972068787, 'epsilon_dpo/beta_margin_grad_std': 0.16221848130226135, 'kl/beta': 0.0010794695699587464, 'kl/avg_steps': 0.6875, 'epoch': 0.85}
|
|||
|
|
85%|█████████████████████████████████████████████████████████████████████████████████████████████████▎ | 581/681 [36:30<04:02, 2.42s/it]
85%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 582/681 [36:33<04:04, 2.47s/it]
{'loss': 0.874, 'grad_norm': 56.42830276489258, 'learning_rate': 3.222175147833556e-08, 'rewards/chosen': -0.5769472718238831, 'rewards/rejected': -1.5062320232391357, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.9292848110198975, 'logps/chosen': -586.5724487304688, 'logps/rejected': -1535.237060546875, 'logps/ref_chosen': -47.183502197265625, 'logps/ref_rejected': -118.7525634765625, 'logits/chosen': -3.625027656555176, 'logits/rejected': -4.7767333984375, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.001064834650605917, 'epsilon_dpo/loss_margin_mean': 877.0955200195312, 'epsilon_dpo/beta_margin_mean': 0.9292847514152527, 'epsilon_dpo/beta_margin_std': 1.0593138933181763, 'epsilon_dpo/beta_margin_grad_mean': -0.32155805826187134, 'epsilon_dpo/beta_margin_grad_std': 0.18931035697460175, 'kl/beta': 0.0010720988502725959, 'kl/avg_steps': 0.6875, 'epoch': 0.85}
|
|||
|
|
85%|█████████████████████████████████████████████████████████████████████████████████████████████████▍ | 582/681 [36:33<04:04, 2.47s/it]
86%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 583/681 [36:36<04:09, 2.55s/it]
{'loss': 1.018, 'grad_norm': 76.16841888427734, 'learning_rate': 3.159440233840763e-08, 'rewards/chosen': -0.5164851546287537, 'rewards/rejected': -1.2378886938095093, 'rewards/accuracies': 0.75, 'rewards/margins': 0.7214035987854004, 'logps/chosen': -522.0079956054688, 'logps/rejected': -1267.432861328125, 'logps/ref_chosen': -37.24636459350586, 'logps/ref_rejected': -95.97724151611328, 'logits/chosen': -3.5470635890960693, 'logits/rejected': -4.620864391326904, 'kl/p_epsilon_steps': 0.734375, 'kl/n_epsilon_steps': 0.265625, 'epsilon_dpo/beta': 0.0010598934022709727, 'epsilon_dpo/loss_margin_mean': 686.6939697265625, 'epsilon_dpo/beta_margin_mean': 0.7214035391807556, 'epsilon_dpo/beta_margin_std': 1.0699379444122314, 'epsilon_dpo/beta_margin_grad_mean': -0.3548807203769684, 'epsilon_dpo/beta_margin_grad_std': 0.19512853026390076, 'kl/beta': 0.0010647785384207964, 'kl/avg_steps': 0.46875, 'epoch': 0.86}
|
|||
|
|
86%|█████████████████████████████████████████████████████████████████████████████████████████████████▌ | 583/681 [36:36<04:09, 2.55s/it]
86%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 584/681 [36:38<04:04, 2.52s/it]
{'loss': 0.7209, 'grad_norm': 45.21849060058594, 'learning_rate': 3.0972808389096635e-08, 'rewards/chosen': -0.3644617795944214, 'rewards/rejected': -1.335336685180664, 'rewards/accuracies': 0.96875, 'rewards/margins': 0.9708749055862427, 'logps/chosen': -381.122314453125, 'logps/rejected': -1377.441650390625, 'logps/ref_chosen': -34.52136993408203, 'logps/ref_rejected': -105.40646362304688, 'logits/chosen': -3.285341262817383, 'logits/rejected': -4.798908710479736, 'kl/p_epsilon_steps': 0.9375, 'kl/n_epsilon_steps': 0.0625, 'epsilon_dpo/beta': 0.0010506423423066735, 'epsilon_dpo/loss_margin_mean': 925.4342041015625, 'epsilon_dpo/beta_margin_mean': 0.9708749651908875, 'epsilon_dpo/beta_margin_std': 0.6488715410232544, 'epsilon_dpo/beta_margin_grad_mean': -0.2919199466705322, 'epsilon_dpo/beta_margin_grad_std': 0.1186746209859848, 'kl/beta': 0.0010598106309771538, 'kl/avg_steps': 0.875, 'epoch': 0.86}
|
|||
|
|
86%|█████████████████████████████████████████████████████████████████████████████████████████████████▊ | 584/681 [36:38<04:04, 2.52s/it]
86%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 585/681 [36:40<03:59, 2.50s/it]
{'loss': 0.8314, 'grad_norm': 49.96684265136719, 'learning_rate': 3.035698600998121e-08, 'rewards/chosen': -0.4328582286834717, 'rewards/rejected': -1.3340110778808594, 'rewards/accuracies': 0.875, 'rewards/margins': 0.9011527895927429, 'logps/chosen': -450.2691345214844, 'logps/rejected': -1373.3876953125, 'logps/ref_chosen': -36.641475677490234, 'logps/ref_rejected': -92.18727111816406, 'logits/chosen': -3.437217950820923, 'logits/rejected': -4.798175811767578, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.001042842399328947, 'epsilon_dpo/loss_margin_mean': 867.5726928710938, 'epsilon_dpo/beta_margin_mean': 0.9011527895927429, 'epsilon_dpo/beta_margin_std': 0.8913306593894958, 'epsilon_dpo/beta_margin_grad_mean': -0.3161095976829529, 'epsilon_dpo/beta_margin_grad_std': 0.1634971797466278, 'kl/beta': 0.001050617778673768, 'kl/avg_steps': 0.75, 'epoch': 0.86}
|
|||
|
|
86%|█████████████████████████████████████████████████████████████████████████████████████████████████▉ | 585/681 [36:41<03:59, 2.50s/it]
86%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 586/681 [36:43<03:54, 2.47s/it]
{'loss': 0.8691, 'grad_norm': 41.283748626708984, 'learning_rate': 2.974695142855388e-08, 'rewards/chosen': -0.508251428604126, 'rewards/rejected': -1.341785192489624, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.833533763885498, 'logps/chosen': -526.1253051757812, 'logps/rejected': -1396.59326171875, 'logps/ref_chosen': -36.78833770751953, 'logps/ref_rejected': -98.72267150878906, 'logits/chosen': -3.5555217266082764, 'logits/rejected': -4.804073333740234, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0010367088252678514, 'epsilon_dpo/loss_margin_mean': 808.5335693359375, 'epsilon_dpo/beta_margin_mean': 0.833533763885498, 'epsilon_dpo/beta_margin_std': 0.8716885447502136, 'epsilon_dpo/beta_margin_grad_mean': -0.3300473392009735, 'epsilon_dpo/beta_margin_grad_std': 0.16417358815670013, 'kl/beta': 0.001042796764522791, 'kl/avg_steps': 0.59375, 'epoch': 0.86}
|
|||
|
|
86%|██████████████████████████████████████████████████████████████████████████████████████████████████ | 586/681 [36:43<03:54, 2.47s/it]
86%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 587/681 [36:46<03:56, 2.51s/it]
{'loss': 0.8854, 'grad_norm': 43.92211151123047, 'learning_rate': 2.9142720719793122e-08, 'rewards/chosen': -0.30333322286605835, 'rewards/rejected': -1.098832130432129, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.7954989075660706, 'logps/chosen': -320.67523193359375, 'logps/rejected': -1156.609130859375, 'logps/ref_chosen': -27.06256103515625, 'logps/ref_rejected': -87.31655883789062, 'logits/chosen': -3.2840447425842285, 'logits/rejected': -4.652719020843506, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.0010302658192813396, 'epsilon_dpo/loss_margin_mean': 775.6798706054688, 'epsilon_dpo/beta_margin_mean': 0.7954989075660706, 'epsilon_dpo/beta_margin_std': 0.8702175617218018, 'epsilon_dpo/beta_margin_grad_mean': -0.33965250849723816, 'epsilon_dpo/beta_margin_grad_std': 0.15424403548240662, 'kl/beta': 0.001036641770042479, 'kl/avg_steps': 0.625, 'epoch': 0.86}
|
|||
|
|
86%|██████████████████████████████████████████████████████████████████████████████████████████████████▎ | 587/681 [36:46<03:56, 2.51s/it]
86%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 588/681 [36:48<03:50, 2.47s/it]
{'loss': 0.8623, 'grad_norm': 54.41194152832031, 'learning_rate': 2.8544309805740018e-08, 'rewards/chosen': -0.45345091819763184, 'rewards/rejected': -1.3257596492767334, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.8723088502883911, 'logps/chosen': -473.547607421875, 'logps/rejected': -1410.73583984375, 'logps/ref_chosen': -32.258827209472656, 'logps/ref_rejected': -112.76737976074219, 'logits/chosen': -3.385420083999634, 'logits/rejected': -4.716907978057861, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0010235445806756616, 'epsilon_dpo/loss_margin_mean': 856.6797485351562, 'epsilon_dpo/beta_margin_mean': 0.8723087906837463, 'epsilon_dpo/beta_margin_std': 0.9220552444458008, 'epsilon_dpo/beta_margin_grad_mean': -0.3219730854034424, 'epsilon_dpo/beta_margin_grad_std': 0.1715155392885208, 'kl/beta': 0.001030202955007553, 'kl/avg_steps': 0.65625, 'epoch': 0.86}
|
|||
|
|
86%|██████████████████████████████████████████████████████████████████████████████████████████████████▍ | 588/681 [36:48<03:50, 2.47s/it]
86%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 589/681 [36:50<03:42, 2.42s/it]
{'loss': 0.846, 'grad_norm': 57.298946380615234, 'learning_rate': 2.7951734455078786e-08, 'rewards/chosen': -0.46269693970680237, 'rewards/rejected': -1.4045993089675903, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.9419023990631104, 'logps/chosen': -488.34820556640625, 'logps/rejected': -1502.165771484375, 'logps/ref_chosen': -36.310951232910156, 'logps/ref_rejected': -119.32230377197266, 'logits/chosen': -3.4220762252807617, 'logits/rejected': -4.901893138885498, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.0010178310330957174, 'epsilon_dpo/loss_margin_mean': 930.8062744140625, 'epsilon_dpo/beta_margin_mean': 0.9419023990631104, 'epsilon_dpo/beta_margin_std': 1.028359293937683, 'epsilon_dpo/beta_margin_grad_mean': -0.3191934823989868, 'epsilon_dpo/beta_margin_grad_std': 0.17622847855091095, 'kl/beta': 0.001023486373014748, 'kl/avg_steps': 0.5625, 'epoch': 0.86}
|
|||
|
|
86%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 589/681 [36:50<03:42, 2.42s/it]
87%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 590/681 [36:53<03:38, 2.40s/it]
{'loss': 0.7904, 'grad_norm': 43.128292083740234, 'learning_rate': 2.736501028272095e-08, 'rewards/chosen': -0.3574645221233368, 'rewards/rejected': -1.302985429763794, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.94552081823349, 'logps/chosen': -384.21209716796875, 'logps/rejected': -1403.730712890625, 'logps/ref_chosen': -31.54564094543457, 'logps/ref_rejected': -112.30560302734375, 'logits/chosen': -3.2722253799438477, 'logits/rejected': -4.7439470291137695, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0010111834853887558, 'epsilon_dpo/loss_margin_mean': 938.7587280273438, 'epsilon_dpo/beta_margin_mean': 0.94552081823349, 'epsilon_dpo/beta_margin_std': 0.8651568293571472, 'epsilon_dpo/beta_margin_grad_mean': -0.3087705075740814, 'epsilon_dpo/beta_margin_grad_std': 0.15020431578159332, 'kl/beta': 0.001017761416733265, 'kl/avg_steps': 0.65625, 'epoch': 0.87}
|
|||
|
|
87%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 590/681 [36:53<03:38, 2.40s/it]
87%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 591/681 [36:55<03:33, 2.37s/it]
{'loss': 0.9023, 'grad_norm': 63.2326545715332, 'learning_rate': 2.678415274939408e-08, 'rewards/chosen': -0.39648881554603577, 'rewards/rejected': -1.211134433746338, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.814645528793335, 'logps/chosen': -425.36102294921875, 'logps/rejected': -1297.699462890625, 'logps/ref_chosen': -32.81976318359375, 'logps/ref_rejected': -89.97746276855469, 'logits/chosen': -3.32479190826416, 'logits/rejected': -4.7041168212890625, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0010042748181149364, 'epsilon_dpo/loss_margin_mean': 815.1807861328125, 'epsilon_dpo/beta_margin_mean': 0.8146455883979797, 'epsilon_dpo/beta_margin_std': 0.9358741641044617, 'epsilon_dpo/beta_margin_grad_mean': -0.3359038829803467, 'epsilon_dpo/beta_margin_grad_std': 0.17261067032814026, 'kl/beta': 0.0010111258598044515, 'kl/avg_steps': 0.6875, 'epoch': 0.87}
|
|||
|
|
87%|██████████████████████████████████████████████████████████████████████████████████████████████████▉ | 591/681 [36:55<03:33, 2.37s/it]
87%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 592/681 [36:57<03:25, 2.31s/it]
{'loss': 0.877, 'grad_norm': 49.01154708862305, 'learning_rate': 2.6209177161234442e-08, 'rewards/chosen': -0.3929889500141144, 'rewards/rejected': -1.1834845542907715, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.7904956340789795, 'logps/chosen': -423.3301696777344, 'logps/rejected': -1268.98046875, 'logps/ref_chosen': -31.739425659179688, 'logps/ref_rejected': -81.3411865234375, 'logits/chosen': -3.4462766647338867, 'logits/rejected': -4.8302812576293945, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.0009980452014133334, 'epsilon_dpo/loss_margin_mean': 796.0484008789062, 'epsilon_dpo/beta_margin_mean': 0.7904956340789795, 'epsilon_dpo/beta_margin_std': 0.8030670285224915, 'epsilon_dpo/beta_margin_grad_mean': -0.33510512113571167, 'epsilon_dpo/beta_margin_grad_std': 0.15517839789390564, 'kl/beta': 0.001004221849143505, 'kl/avg_steps': 0.625, 'epoch': 0.87}
|
|||
|
|
87%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 592/681 [36:57<03:25, 2.31s/it]
87%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 593/681 [37:00<03:30, 2.39s/it]
{'loss': 0.9905, 'grad_norm': 50.16069412231445, 'learning_rate': 2.564009866938349e-08, 'rewards/chosen': -0.35422414541244507, 'rewards/rejected': -0.965508759021759, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.611284613609314, 'logps/chosen': -388.3526611328125, 'logps/rejected': -1040.68310546875, 'logps/ref_chosen': -33.75498962402344, 'logps/ref_rejected': -66.24615478515625, 'logits/chosen': -3.442495822906494, 'logits/rejected': -4.646967887878418, 'kl/p_epsilon_steps': 0.734375, 'kl/n_epsilon_steps': 0.265625, 'epsilon_dpo/beta': 0.0009934057015925646, 'epsilon_dpo/loss_margin_mean': 619.8391723632812, 'epsilon_dpo/beta_margin_mean': 0.611284613609314, 'epsilon_dpo/beta_margin_std': 0.7558280825614929, 'epsilon_dpo/beta_margin_grad_mean': -0.3691524267196655, 'epsilon_dpo/beta_margin_grad_std': 0.15739652514457703, 'kl/beta': 0.0009979844326153398, 'kl/avg_steps': 0.46875, 'epoch': 0.87}
|
|||
|
|
87%|███████████████████████████████████████████████████████████████████████████████████████████████████▎ | 593/681 [37:00<03:30, 2.39s/it]
87%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 594/681 [37:02<03:29, 2.41s/it]
{'loss': 0.9111, 'grad_norm': 56.66592025756836, 'learning_rate': 2.5076932269588708e-08, 'rewards/chosen': -0.40900394320487976, 'rewards/rejected': -1.1389310359954834, 'rewards/accuracies': 0.78125, 'rewards/margins': 0.7299271821975708, 'logps/chosen': -447.0162353515625, 'logps/rejected': -1248.580322265625, 'logps/ref_chosen': -34.56220245361328, 'logps/ref_rejected': -92.779052734375, 'logits/chosen': -3.383502960205078, 'logits/rejected': -4.847668647766113, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.0009878396522253752, 'epsilon_dpo/loss_margin_mean': 743.34716796875, 'epsilon_dpo/beta_margin_mean': 0.7299271821975708, 'epsilon_dpo/beta_margin_std': 0.7822921276092529, 'epsilon_dpo/beta_margin_grad_mean': -0.3453194499015808, 'epsilon_dpo/beta_margin_grad_std': 0.15376316010951996, 'kl/beta': 0.0009933282854035497, 'kl/avg_steps': 0.5625, 'epoch': 0.87}
|
|||
|
|
87%|███████████████████████████████████████████████████████████████████████████████████████████████████▍ | 594/681 [37:02<03:29, 2.41s/it]
87%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 595/681 [37:05<03:39, 2.55s/it]
{'loss': 0.9122, 'grad_norm': 48.688880920410156, 'learning_rate': 2.451969280180849e-08, 'rewards/chosen': -0.3539902865886688, 'rewards/rejected': -1.0759739875793457, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.7219836711883545, 'logps/chosen': -390.6729431152344, 'logps/rejected': -1183.484619140625, 'logps/ref_chosen': -31.321365356445312, 'logps/ref_rejected': -85.45933532714844, 'logits/chosen': -3.31217098236084, 'logits/rejected': -4.6105427742004395, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0009810791816562414, 'epsilon_dpo/loss_margin_mean': 738.6736450195312, 'epsilon_dpo/beta_margin_mean': 0.7219837307929993, 'epsilon_dpo/beta_margin_std': 0.8028604388237, 'epsilon_dpo/beta_margin_grad_mean': -0.35142308473587036, 'epsilon_dpo/beta_margin_grad_std': 0.14063841104507446, 'kl/beta': 0.0009877720149233937, 'kl/avg_steps': 0.6875, 'epoch': 0.87}
|
|||
|
|
87%|███████████████████████████████████████████████████████████████████████████████████████████████████▌ | 595/681 [37:05<03:39, 2.55s/it]
88%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 596/681 [37:07<03:27, 2.44s/it]
{'loss': 0.9049, 'grad_norm': 47.94287872314453, 'learning_rate': 2.396839494982103e-08, 'rewards/chosen': -0.3397039771080017, 'rewards/rejected': -1.0443021059036255, 'rewards/accuracies': 0.875, 'rewards/margins': 0.7045981287956238, 'logps/chosen': -378.280517578125, 'logps/rejected': -1162.1795654296875, 'logps/ref_chosen': -30.307376861572266, 'logps/ref_rejected': -88.15638732910156, 'logits/chosen': -3.158041000366211, 'logits/rejected': -4.6905059814453125, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0009740737732499838, 'epsilon_dpo/loss_margin_mean': 726.050048828125, 'epsilon_dpo/beta_margin_mean': 0.7045981287956238, 'epsilon_dpo/beta_margin_std': 0.7173324227333069, 'epsilon_dpo/beta_margin_grad_mean': -0.3495016098022461, 'epsilon_dpo/beta_margin_grad_std': 0.13467830419540405, 'kl/beta': 0.0009810274932533503, 'kl/avg_steps': 0.71875, 'epoch': 0.88}
|
|||
|
|
88%|███████████████████████████████████████████████████████████████████████████████████████████████████▊ | 596/681 [37:07<03:27, 2.44s/it]
88%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 597/681 [37:10<03:26, 2.46s/it]
{'loss': 0.8543, 'grad_norm': 41.356510162353516, 'learning_rate': 2.3423053240837514e-08, 'rewards/chosen': -0.3919217586517334, 'rewards/rejected': -1.2861260175704956, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.8942043781280518, 'logps/chosen': -437.45770263671875, 'logps/rejected': -1438.8807373046875, 'logps/ref_chosen': -34.63069152832031, 'logps/ref_rejected': -107.39651489257812, 'logits/chosen': -3.414703845977783, 'logits/rejected': -4.802955627441406, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0009677313501015306, 'epsilon_dpo/loss_margin_mean': 928.6572265625, 'epsilon_dpo/beta_margin_mean': 0.894204318523407, 'epsilon_dpo/beta_margin_std': 0.9434449076652527, 'epsilon_dpo/beta_margin_grad_mean': -0.32157784700393677, 'epsilon_dpo/beta_margin_grad_std': 0.17215141654014587, 'kl/beta': 0.0009740266250446439, 'kl/avg_steps': 0.65625, 'epoch': 0.88}
|
|||
|
|
88%|███████████████████████████████████████████████████████████████████████████████████████████████████▉ | 597/681 [37:10<03:26, 2.46s/it]
88%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 598/681 [37:12<03:21, 2.43s/it]
{'loss': 0.9145, 'grad_norm': 63.38301467895508, 'learning_rate': 2.2883682045119062e-08, 'rewards/chosen': -0.40632039308547974, 'rewards/rejected': -1.1718735694885254, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.7655530571937561, 'logps/chosen': -458.43048095703125, 'logps/rejected': -1317.583740234375, 'logps/ref_chosen': -38.996089935302734, 'logps/ref_rejected': -97.60490417480469, 'logits/chosen': -3.349184989929199, 'logits/rejected': -4.659879684448242, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0009614220471121371, 'epsilon_dpo/loss_margin_mean': 800.5443725585938, 'epsilon_dpo/beta_margin_mean': 0.7655530571937561, 'epsilon_dpo/beta_margin_std': 0.8700756430625916, 'epsilon_dpo/beta_margin_grad_mean': -0.34296542406082153, 'epsilon_dpo/beta_margin_grad_std': 0.16591456532478333, 'kl/beta': 0.0009676762856543064, 'kl/avg_steps': 0.65625, 'epoch': 0.88}
|
|||
|
|
88%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 598/681 [37:12<03:21, 2.43s/it]
88%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 599/681 [37:15<03:21, 2.46s/it]
{'loss': 0.915, 'grad_norm': 51.95116424560547, 'learning_rate': 2.2350295575598367e-08, 'rewards/chosen': -0.3669903874397278, 'rewards/rejected': -1.0654559135437012, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.6984655857086182, 'logps/chosen': -416.742431640625, 'logps/rejected': -1206.987060546875, 'logps/ref_chosen': -33.18586730957031, 'logps/ref_rejected': -88.57789611816406, 'logits/chosen': -3.448948621749878, 'logits/rejected': -4.653861045837402, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0009536515572108328, 'epsilon_dpo/loss_margin_mean': 734.8526000976562, 'epsilon_dpo/beta_margin_mean': 0.6984655857086182, 'epsilon_dpo/beta_margin_std': 0.7249807715415955, 'epsilon_dpo/beta_margin_grad_mean': -0.34948039054870605, 'epsilon_dpo/beta_margin_grad_std': 0.14211682975292206, 'kl/beta': 0.0009613672737032175, 'kl/avg_steps': 0.8125, 'epoch': 0.88}
|
|||
|
|
88%|████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 599/681 [37:15<03:21, 2.46s/it]
88%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 600/681 [37:17<03:21, 2.49s/it]
{'loss': 0.8895, 'grad_norm': 49.26176834106445, 'learning_rate': 2.1822907887504932e-08, 'rewards/chosen': -0.39791154861450195, 'rewards/rejected': -1.1117727756500244, 'rewards/accuracies': 0.875, 'rewards/margins': 0.7138612270355225, 'logps/chosen': -460.50970458984375, 'logps/rejected': -1264.110595703125, 'logps/ref_chosen': -42.37248229980469, 'logps/ref_rejected': -89.61239624023438, 'logits/chosen': -3.4161376953125, 'logits/rejected': -4.741451263427734, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0009474557591602206, 'epsilon_dpo/loss_margin_mean': 756.361083984375, 'epsilon_dpo/beta_margin_mean': 0.7138612270355225, 'epsilon_dpo/beta_margin_std': 0.6741172075271606, 'epsilon_dpo/beta_margin_grad_mean': -0.34580788016319275, 'epsilon_dpo/beta_margin_grad_std': 0.13170255720615387, 'kl/beta': 0.000953619135543704, 'kl/avg_steps': 0.65625, 'epoch': 0.88}
|
|||
|
|
88%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 600/681 [37:17<03:21, 2.49s/it][INFO|trainer.py:4307] 2026-04-18 10:08:39,549 >>
|
|||
|
|
***** Running Evaluation *****
|
|||
|
|
[INFO|trainer.py:4309] 2026-04-18 10:08:39,549 >> Num examples = 2339
|
|||
|
|
[INFO|trainer.py:4312] 2026-04-18 10:08:39,549 >> Batch size = 8
|
|||
|
|
|
|||
|
|
0%| | 0/73 [00:00<?, ?it/s][A
|
|||
|
|
3%|███▏ | 2/73 [00:00<00:18, 3.84it/s][A
|
|||
|
|
4%|████▊ | 3/73 [00:01<00:26, 2.60it/s][A
|
|||
|
|
5%|██████▍ | 4/73 [00:01<00:30, 2.26it/s][A
|
|||
|
|
7%|████████ | 5/73 [00:02<00:32, 2.10it/s][A
|
|||
|
|
8%|█████████▌ | 6/73 [00:02<00:33, 2.03it/s][A
|
|||
|
|
10%|███████████▏ | 7/73 [00:03<00:32, 2.02it/s][A
|
|||
|
|
11%|████████████▊ | 8/73 [00:03<00:31, 2.07it/s][A
|
|||
|
|
12%|██████████████▍ | 9/73 [00:04<00:32, 1.99it/s][A
|
|||
|
|
14%|███████████████▉ | 10/73 [00:04<00:32, 1.95it/s][A
|
|||
|
|
15%|█████████████████▍ | 11/73 [00:05<00:31, 1.96it/s][A
|
|||
|
|
16%|███████████████████ | 12/73 [00:05<00:31, 1.96it/s][A
|
|||
|
|
18%|████████████████████▋ | 13/73 [00:06<00:31, 1.91it/s][A
|
|||
|
|
19%|██████████████████████▏ | 14/73 [00:06<00:30, 1.91it/s][A
|
|||
|
|
21%|███████████████████████▊ | 15/73 [00:07<00:31, 1.87it/s][A
|
|||
|
|
22%|█████████████████████████▍ | 16/73 [00:07<00:30, 1.86it/s][A
|
|||
|
|
23%|███████████████████████████ | 17/73 [00:08<00:29, 1.88it/s][A
|
|||
|
|
25%|████████████████████████████▌ | 18/73 [00:09<00:29, 1.85it/s][A
|
|||
|
|
26%|██████████████████████████████▏ | 19/73 [00:09<00:29, 1.85it/s][A
|
|||
|
|
27%|███████████████████████████████▊ | 20/73 [00:10<00:29, 1.83it/s][A
|
|||
|
|
29%|█████████████████████████████████▎ | 21/73 [00:10<00:28, 1.83it/s][A
|
|||
|
|
30%|██████████████████████████████████▉ | 22/73 [00:11<00:27, 1.82it/s][A
|
|||
|
|
32%|████████████████████████████████████▌ | 23/73 [00:11<00:27, 1.85it/s][A
|
|||
|
|
33%|██████████████████████████████████████▏ | 24/73 [00:12<00:26, 1.88it/s][A
|
|||
|
|
34%|███████████████████████████████████████▋ | 25/73 [00:12<00:25, 1.87it/s][A
|
|||
|
|
36%|█████████████████████████████████████████▎ | 26/73 [00:13<00:24, 1.90it/s][A
|
|||
|
|
37%|██████████████████████████████████████████▉ | 27/73 [00:13<00:22, 2.05it/s][A
|
|||
|
|
38%|████████████████████████████████████████████▍ | 28/73 [00:14<00:22, 2.04it/s][A
|
|||
|
|
40%|██████████████████████████████████████████████ | 29/73 [00:14<00:21, 2.01it/s][A
|
|||
|
|
41%|███████████████████████████████████████████████▋ | 30/73 [00:15<00:21, 1.96it/s][A
|
|||
|
|
42%|█████████████████████████████████████████████████▎ | 31/73 [00:15<00:21, 1.92it/s][A
|
|||
|
|
44%|██████████████████████████████████████████████████▊ | 32/73 [00:16<00:21, 1.94it/s][A
|
|||
|
|
45%|████████████████████████████████████████████████████▍ | 33/73 [00:16<00:20, 1.93it/s][A
|
|||
|
|
47%|██████████████████████████████████████████████████████ | 34/73 [00:17<00:19, 1.97it/s][A
|
|||
|
|
48%|███████████████████████████████████████████████████████▌ | 35/73 [00:17<00:19, 1.91it/s][A
|
|||
|
|
49%|█████████████████████████████████████████████████████████▏ | 36/73 [00:18<00:19, 1.92it/s][A
|
|||
|
|
51%|██████████████████████████████████████████████████████████▊ | 37/73 [00:18<00:19, 1.87it/s][A
|
|||
|
|
52%|████████████████████████████████████████████████████████████▍ | 38/73 [00:19<00:18, 1.93it/s][A
|
|||
|
|
53%|█████████████████████████████████████████████████████████████▉ | 39/73 [00:19<00:17, 1.92it/s][A
|
|||
|
|
55%|███████████████████████████████████████████████████████████████▌ | 40/73 [00:20<00:17, 1.92it/s][A
|
|||
|
|
56%|█████████████████████████████████████████████████████████████████▏ | 41/73 [00:20<00:16, 1.93it/s][A
|
|||
|
|
58%|██████████████████████████████████████████████████████████████████▋ | 42/73 [00:21<00:16, 1.90it/s][A
|
|||
|
|
59%|████████████████████████████████████████████████████████████████████▎ | 43/73 [00:22<00:15, 1.88it/s][A
|
|||
|
|
60%|█████████████████████████████████████████████████████████████████████▉ | 44/73 [00:22<00:15, 1.86it/s][A
|
|||
|
|
62%|███████████████████████████████████████████████████████████████████████▌ | 45/73 [00:23<00:15, 1.83it/s][A
|
|||
|
|
63%|█████████████████████████████████████████████████████████████████████████ | 46/73 [00:23<00:14, 1.87it/s][A
|
|||
|
|
64%|██████████████████████████████████████████████████████████████████████████▋ | 47/73 [00:24<00:13, 1.88it/s][A
|
|||
|
|
66%|████████████████████████████████████████████████████████████████████████████▎ | 48/73 [00:24<00:12, 1.93it/s][A
|
|||
|
|
67%|█████████████████████████████████████████████████████████████████████████████▊ | 49/73 [00:25<00:12, 1.88it/s][A
|
|||
|
|
68%|███████████████████████████████████████████████████████████████████████████████▍ | 50/73 [00:25<00:12, 1.86it/s][A
|
|||
|
|
70%|█████████████████████████████████████████████████████████████████████████████████ | 51/73 [00:26<00:11, 1.84it/s][A
|
|||
|
|
71%|██████████████████████████████████████████████████████████████████████████████████▋ | 52/73 [00:26<00:11, 1.82it/s][A
|
|||
|
|
73%|████████████████████████████████████████████████████████████████████████████████████▏ | 53/73 [00:27<00:10, 1.94it/s][A
|
|||
|
|
74%|█████████████████████████████████████████████████████████████████████████████████████▊ | 54/73 [00:27<00:09, 1.97it/s][A
|
|||
|
|
75%|███████████████████████████████████████████████████████████████████████████████████████▍ | 55/73 [00:28<00:08, 2.01it/s][A
|
|||
|
|
77%|████████████████████████████████████████████████████████████████████████████████████████▉ | 56/73 [00:28<00:08, 1.96it/s][A
|
|||
|
|
78%|██████████████████████████████████████████████████████████████████████████████████████████▌ | 57/73 [00:29<00:08, 1.91it/s][A
|
|||
|
|
79%|████████████████████████████████████████████████████████████████████████████████████████████▏ | 58/73 [00:29<00:07, 1.94it/s][A
|
|||
|
|
81%|█████████████████████████████████████████████████████████████████████████████████████████████▊ | 59/73 [00:30<00:07, 1.88it/s][A
|
|||
|
|
82%|███████████████████████████████████████████████████████████████████████████████████████████████▎ | 60/73 [00:31<00:06, 1.88it/s][A
|
|||
|
|
84%|████████████████████████████████████████████████████████████████████████████████████████████████▉ | 61/73 [00:31<00:06, 1.87it/s][A
|
|||
|
|
85%|██████████████████████████████████████████████████████████████████████████████████████████████████▌ | 62/73 [00:32<00:05, 1.85it/s][A
|
|||
|
|
86%|████████████████████████████████████████████████████████████████████████████████████████████████████ | 63/73 [00:32<00:05, 1.90it/s][A
|
|||
|
|
88%|█████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 64/73 [00:33<00:04, 1.92it/s][A
|
|||
|
|
89%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 65/73 [00:33<00:04, 1.87it/s][A
|
|||
|
|
90%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 66/73 [00:34<00:03, 1.86it/s][A
|
|||
|
|
92%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 67/73 [00:34<00:03, 1.85it/s][A
|
|||
|
|
93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 68/73 [00:35<00:02, 1.86it/s][A
|
|||
|
|
95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 69/73 [00:35<00:02, 1.97it/s][A
|
|||
|
|
96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 70/73 [00:36<00:01, 1.92it/s][A
|
|||
|
|
97%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 71/73 [00:36<00:01, 1.90it/s][A
|
|||
|
|
99%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 72/73 [00:37<00:00, 1.88it/s][A
|
|||
|
|
100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 73/73 [00:37<00:00, 1.99it/s][A
|
|||
|
|
[A{'eval_loss': 0.5823308825492859, 'eval_runtime': 38.382, 'eval_samples_per_second': 60.94, 'eval_steps_per_second': 1.928, 'eval_epsilon_dpo/beta': 0.0009440449066460133, 'eval_epsilon_dpo/loss_margin_mean': 478.8302001953125, 'eval_epsilon_dpo/beta_margin_mean': 0.44606366753578186, 'eval_epsilon_dpo/beta_margin_std': 0.8790761828422546, 'eval_epsilon_dpo/beta_margin_grad_mean': -0.40722984075546265, 'eval_epsilon_dpo/beta_margin_grad_std': 0.1766805797815323, 'eval_rewards/chosen': -0.6674680709838867, 'eval_rewards/rejected': -1.1135317087173462, 'eval_rewards/accuracies': 0.7037671208381653, 'eval_rewards/margins': 0.44606366753578186, 'eval_logps/chosen': -771.9093627929688, 'eval_logps/rejected': -1274.5286865234375, 'eval_logps/ref_chosen': -68.29110717773438, 'eval_logps/ref_rejected': -92.08038330078125, 'eval_logits/chosen': -3.8786203861236572, 'eval_logits/rejected': -4.723351955413818, 'eval_kl/p_epsilon_steps': 0.6819349527359009, 'eval_kl/n_epsilon_steps': 0.31763699650764465, 'epoch': 0.88}
|
|||
|
|
88%|████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 600/681 [37:56<03:21, 2.49s/it]
|
|||
|
|
100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 73/73 [00:37<00:00, 1.99it/s][A
|
|||
|
|
[A[INFO|trainer.py:3984] 2026-04-18 10:09:33,938 >> Saving model checkpoint to /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-600
|
|||
|
|
[INFO|configuration_utils.py:419] 2026-04-18 10:09:33,959 >> Configuration saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-600/config.json
|
|||
|
|
[INFO|configuration_utils.py:911] 2026-04-18 10:09:33,976 >> Configuration saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-600/generation_config.json
|
|||
|
|
[INFO|modeling_utils.py:3580] 2026-04-18 10:10:31,283 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 6 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-600/model.safetensors.index.json.
|
|||
|
|
[INFO|tokenization_utils_base.py:2510] 2026-04-18 10:10:31,322 >> tokenizer config file saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-600/tokenizer_config.json
|
|||
|
|
[INFO|tokenization_utils_base.py:2519] 2026-04-18 10:10:31,334 >> Special tokens file saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-600/special_tokens_map.json
|
|||
|
|
[INFO|trainer.py:4083] 2026-04-18 10:13:53,767 >> Deleting older checkpoint [/scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-200] due to args.save_total_limit
|
|||
|
|
88%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 601/681 [42:36<2:09:47, 97.35s/it]
{'loss': 0.852, 'grad_norm': 48.359046936035156, 'learning_rate': 2.1301532877994742e-08, 'rewards/chosen': -0.35252028703689575, 'rewards/rejected': -1.2315044403076172, 'rewards/accuracies': 0.875, 'rewards/margins': 0.8789842128753662, 'logps/chosen': -406.67852783203125, 'logps/rejected': -1410.8968505859375, 'logps/ref_chosen': -34.16810607910156, 'logps/ref_rejected': -99.91683959960938, 'logits/chosen': -3.238035202026367, 'logits/rejected': -4.882007122039795, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0009403903386555612, 'epsilon_dpo/loss_margin_mean': 938.4696044921875, 'epsilon_dpo/beta_margin_mean': 0.8789842128753662, 'epsilon_dpo/beta_margin_std': 0.901688814163208, 'epsilon_dpo/beta_margin_grad_mean': -0.32094281911849976, 'epsilon_dpo/beta_margin_grad_std': 0.16856704652309418, 'kl/beta': 0.0009474018006585538, 'kl/avg_steps': 0.75, 'epoch': 0.88}
|
|||
|
|
88%|██████████████████████████████████████████████████████████████████████████████████████████████████▊ | 601/681 [42:36<2:09:47, 97.35s/it]
88%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 602/681 [42:38<1:30:45, 68.93s/it]
{'loss': 0.8003, 'grad_norm': 48.63134765625, 'learning_rate': 2.0786184285784298e-08, 'rewards/chosen': -0.25088733434677124, 'rewards/rejected': -1.12050199508667, 'rewards/accuracies': 0.953125, 'rewards/margins': 0.8696146011352539, 'logps/chosen': -303.0639343261719, 'logps/rejected': -1296.177978515625, 'logps/ref_chosen': -34.405120849609375, 'logps/ref_rejected': -93.47988891601562, 'logits/chosen': -3.2103443145751953, 'logits/rejected': -4.763635635375977, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0009328021551482379, 'epsilon_dpo/loss_margin_mean': 934.039306640625, 'epsilon_dpo/beta_margin_mean': 0.8696146011352539, 'epsilon_dpo/beta_margin_std': 0.728697657585144, 'epsilon_dpo/beta_margin_grad_mean': -0.3170680105686188, 'epsilon_dpo/beta_margin_grad_std': 0.13237425684928894, 'kl/beta': 0.0009403491858392954, 'kl/avg_steps': 0.8125, 'epoch': 0.88}
|
|||
|
|
88%|███████████████████████████████████████████████████████████████████████████████████████████████████ | 602/681 [42:38<1:30:45, 68.93s/it]
89%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 603/681 [42:41<1:03:44, 49.03s/it]
{'loss': 0.9099, 'grad_norm': 51.643531799316406, 'learning_rate': 2.0276875690788204e-08, 'rewards/chosen': -0.38833025097846985, 'rewards/rejected': -1.1105290651321411, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.7221988439559937, 'logps/chosen': -470.8554992675781, 'logps/rejected': -1308.265380859375, 'logps/ref_chosen': -53.07399368286133, 'logps/ref_rejected': -107.52302551269531, 'logits/chosen': -3.2944273948669434, 'logits/rejected': -4.612957000732422, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0009261587401852012, 'epsilon_dpo/loss_margin_mean': 782.9607543945312, 'epsilon_dpo/beta_margin_mean': 0.7221988439559937, 'epsilon_dpo/beta_margin_std': 0.7671189904212952, 'epsilon_dpo/beta_margin_grad_mean': -0.3474172055721283, 'epsilon_dpo/beta_margin_grad_std': 0.14739488065242767, 'kl/beta': 0.0009327704319730401, 'kl/avg_steps': 0.71875, 'epoch': 0.89}
|
|||
|
|
89%|███████████████████████████████████████████████████████████████████████████████████████████████████▏ | 603/681 [42:41<1:03:44, 49.03s/it]
89%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 604/681 [42:44<45:03, 35.11s/it]
{'loss': 0.8561, 'grad_norm': 44.274085998535156, 'learning_rate': 1.977362051376158e-08, 'rewards/chosen': -0.3594130277633667, 'rewards/rejected': -1.2111127376556396, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.8516995906829834, 'logps/chosen': -422.5206298828125, 'logps/rejected': -1419.331298828125, 'logps/ref_chosen': -32.21878433227539, 'logps/ref_rejected': -99.47515106201172, 'logits/chosen': -3.4287848472595215, 'logits/rejected': -4.8621602058410645, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0009189706179313362, 'epsilon_dpo/loss_margin_mean': 929.5543823242188, 'epsilon_dpo/beta_margin_mean': 0.8516996502876282, 'epsilon_dpo/beta_margin_std': 0.9336607456207275, 'epsilon_dpo/beta_margin_grad_mean': -0.3316219747066498, 'epsilon_dpo/beta_margin_grad_std': 0.14688213169574738, 'kl/beta': 0.0009261139784939587, 'kl/avg_steps': 0.78125, 'epoch': 0.89}
|
|||
|
|
89%|█████████████████████████████████████████████████████████████████████████████████████████████████████ | 604/681 [42:44<45:03, 35.11s/it]
89%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 605/681 [42:46<32:06, 25.35s/it]
{'loss': 0.882, 'grad_norm': 41.927146911621094, 'learning_rate': 1.9276432015946446e-08, 'rewards/chosen': -0.3291019797325134, 'rewards/rejected': -1.110598087310791, 'rewards/accuracies': 0.875, 'rewards/margins': 0.7814960479736328, 'logps/chosen': -402.01544189453125, 'logps/rejected': -1326.990234375, 'logps/ref_chosen': -42.914276123046875, 'logps/ref_rejected': -108.40269470214844, 'logits/chosen': -3.3077027797698975, 'logits/rejected': -4.713131904602051, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0009129956015385687, 'epsilon_dpo/loss_margin_mean': 859.486328125, 'epsilon_dpo/beta_margin_mean': 0.7814961075782776, 'epsilon_dpo/beta_margin_std': 0.8144083619117737, 'epsilon_dpo/beta_margin_grad_mean': -0.33933547139167786, 'epsilon_dpo/beta_margin_grad_std': 0.15050913393497467, 'kl/beta': 0.0009189348202198744, 'kl/avg_steps': 0.65625, 'epoch': 0.89}
|
|||
|
|
89%|█████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 605/681 [42:46<32:06, 25.35s/it]
89%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 606/681 [42:49<23:07, 18.50s/it]
{'loss': 0.9904, 'grad_norm': 57.995399475097656, 'learning_rate': 1.8785323298722093e-08, 'rewards/chosen': -0.4276297986507416, 'rewards/rejected': -1.045759916305542, 'rewards/accuracies': 0.828125, 'rewards/margins': 0.618130087852478, 'logps/chosen': -506.50384521484375, 'logps/rejected': -1257.863037109375, 'logps/ref_chosen': -37.19722366333008, 'logps/ref_rejected': -102.87519836425781, 'logits/chosen': -3.486124277114868, 'logits/rejected': -4.584151268005371, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.0009078990551643074, 'epsilon_dpo/loss_margin_mean': 685.6812133789062, 'epsilon_dpo/beta_margin_mean': 0.618130087852478, 'epsilon_dpo/beta_margin_std': 0.7797470688819885, 'epsilon_dpo/beta_margin_grad_mean': -0.3660072982311249, 'epsilon_dpo/beta_margin_grad_std': 0.15605977177619934, 'kl/beta': 0.000912943622097373, 'kl/avg_steps': 0.5625, 'epoch': 0.89}
|
|||
|
|
89%|█████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 606/681 [42:49<23:07, 18.50s/it]
89%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 607/681 [42:51<17:00, 13.79s/it]
{'loss': 0.9843, 'grad_norm': 55.2100830078125, 'learning_rate': 1.8300307303259904e-08, 'rewards/chosen': -0.4154084026813507, 'rewards/rejected': -1.046553611755371, 'rewards/accuracies': 0.796875, 'rewards/margins': 0.6311452984809875, 'logps/chosen': -499.9947509765625, 'logps/rejected': -1246.0013427734375, 'logps/ref_chosen': -43.06529235839844, 'logps/ref_rejected': -84.84536743164062, 'logits/chosen': -3.4703445434570312, 'logits/rejected': -4.856854438781738, 'kl/p_epsilon_steps': 0.765625, 'kl/n_epsilon_steps': 0.234375, 'epsilon_dpo/beta': 0.000903104490134865, 'epsilon_dpo/loss_margin_mean': 704.2265014648438, 'epsilon_dpo/beta_margin_mean': 0.6311452984809875, 'epsilon_dpo/beta_margin_std': 0.7819616198539734, 'epsilon_dpo/beta_margin_grad_mean': -0.36472412943840027, 'epsilon_dpo/beta_margin_grad_std': 0.16157901287078857, 'kl/beta': 0.0009078370640054345, 'kl/avg_steps': 0.53125, 'epoch': 0.89}
|
|||
|
|
89%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 607/681 [42:52<17:00, 13.79s/it]
89%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 608/681 [42:54<12:39, 10.40s/it]
{'loss': 0.843, 'grad_norm': 60.484779357910156, 'learning_rate': 1.7821396810182437e-08, 'rewards/chosen': -0.2668021619319916, 'rewards/rejected': -1.0676581859588623, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.8008559942245483, 'logps/chosen': -324.84686279296875, 'logps/rejected': -1294.824462890625, 'logps/ref_chosen': -27.870777130126953, 'logps/ref_rejected': -101.65553283691406, 'logits/chosen': -3.1860623359680176, 'logits/rejected': -4.672609806060791, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0008960742270573974, 'epsilon_dpo/loss_margin_mean': 896.19287109375, 'epsilon_dpo/beta_margin_mean': 0.8008559942245483, 'epsilon_dpo/beta_margin_std': 0.7176367044448853, 'epsilon_dpo/beta_margin_grad_mean': -0.3294691741466522, 'epsilon_dpo/beta_margin_grad_std': 0.13619713485240936, 'kl/beta': 0.0009030396468006074, 'kl/avg_steps': 0.78125, 'epoch': 0.89}
|
|||
|
|
89%|█████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 608/681 [42:54<12:39, 10.40s/it]
89%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 609/681 [42:57<09:40, 8.06s/it]
{'loss': 0.829, 'grad_norm': 52.808658599853516, 'learning_rate': 1.7348604439226617e-08, 'rewards/chosen': -0.3561561107635498, 'rewards/rejected': -1.166949987411499, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.8107938766479492, 'logps/chosen': -432.412841796875, 'logps/rejected': -1410.12060546875, 'logps/ref_chosen': -33.51665496826172, 'logps/ref_rejected': -96.93180084228516, 'logits/chosen': -3.266610860824585, 'logits/rejected': -4.786360740661621, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0008896880317479372, 'epsilon_dpo/loss_margin_mean': 914.2926025390625, 'epsilon_dpo/beta_margin_mean': 0.8107938766479492, 'epsilon_dpo/beta_margin_std': 0.689571738243103, 'epsilon_dpo/beta_margin_grad_mean': -0.3255755305290222, 'epsilon_dpo/beta_margin_grad_std': 0.13147076964378357, 'kl/beta': 0.0008960393606685102, 'kl/avg_steps': 0.71875, 'epoch': 0.89}
|
|||
|
|
89%|█████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 609/681 [42:57<09:40, 8.06s/it]
90%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 610/681 [42:59<07:37, 6.44s/it]
{'loss': 0.9283, 'grad_norm': 50.597049713134766, 'learning_rate': 1.6881942648911074e-08, 'rewards/chosen': -0.34177446365356445, 'rewards/rejected': -1.1019885540008545, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.7602142095565796, 'logps/chosen': -423.406982421875, 'logps/rejected': -1337.302490234375, 'logps/ref_chosen': -39.733856201171875, 'logps/ref_rejected': -88.57766723632812, 'logits/chosen': -3.319277286529541, 'logits/rejected': -4.948149681091309, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.0008841731469146907, 'epsilon_dpo/loss_margin_mean': 865.0516967773438, 'epsilon_dpo/beta_margin_mean': 0.7602141499519348, 'epsilon_dpo/beta_margin_std': 0.892071008682251, 'epsilon_dpo/beta_margin_grad_mean': -0.34196192026138306, 'epsilon_dpo/beta_margin_grad_std': 0.17104457318782806, 'kl/beta': 0.0008896450162865222, 'kl/avg_steps': 0.625, 'epoch': 0.9}
|
|||
|
|
90%|██████████████████████████████████████████████████████████████████████████████████████████████████████ | 610/681 [42:59<07:37, 6.44s/it]
90%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 611/681 [43:02<06:11, 5.31s/it]
{'loss': 0.9188, 'grad_norm': 54.84195327758789, 'learning_rate': 1.6421423736208e-08, 'rewards/chosen': -0.3749154508113861, 'rewards/rejected': -1.1266663074493408, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.7517509460449219, 'logps/chosen': -460.02740478515625, 'logps/rejected': -1376.174072265625, 'logps/ref_chosen': -34.78019332885742, 'logps/ref_rejected': -90.61834716796875, 'logits/chosen': -3.4497299194335938, 'logits/rejected': -4.820193290710449, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0008777128532528877, 'epsilon_dpo/loss_margin_mean': 860.3085327148438, 'epsilon_dpo/beta_margin_mean': 0.7517509460449219, 'epsilon_dpo/beta_margin_std': 0.8516819477081299, 'epsilon_dpo/beta_margin_grad_mean': -0.3441890478134155, 'epsilon_dpo/beta_margin_grad_std': 0.1642007678747177, 'kl/beta': 0.0008841192466206849, 'kl/avg_steps': 0.734375, 'epoch': 0.9}
|
|||
|
|
90%|██████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 611/681 [43:02<06:11, 5.31s/it]
90%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 612/681 [43:04<05:07, 4.45s/it]
{'loss': 0.8826, 'grad_norm': 41.552490234375, 'learning_rate': 1.5967059836219042e-08, 'rewards/chosen': -0.3019838333129883, 'rewards/rejected': -1.0831260681152344, 'rewards/accuracies': 0.875, 'rewards/margins': 0.7811422348022461, 'logps/chosen': -380.4996337890625, 'logps/rejected': -1338.08837890625, 'logps/ref_chosen': -35.333831787109375, 'logps/ref_rejected': -93.14432525634766, 'logits/chosen': -3.163311004638672, 'logits/rejected': -4.886482238769531, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0008717270102351904, 'epsilon_dpo/loss_margin_mean': 899.7782592773438, 'epsilon_dpo/beta_margin_mean': 0.7811422348022461, 'epsilon_dpo/beta_margin_std': 0.8147025108337402, 'epsilon_dpo/beta_margin_grad_mean': -0.33847570419311523, 'epsilon_dpo/beta_margin_grad_std': 0.15062828361988068, 'kl/beta': 0.0008776738541200757, 'kl/avg_steps': 0.6875, 'epoch': 0.9}
|
|||
|
|
90%|██████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 612/681 [43:04<05:07, 4.45s/it]
90%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 613/681 [43:07<04:24, 3.90s/it]
{'loss': 0.7942, 'grad_norm': 61.14183044433594, 'learning_rate': 1.551886292185553e-08, 'rewards/chosen': -0.33134663105010986, 'rewards/rejected': -1.1872930526733398, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.8559463620185852, 'logps/chosen': -418.6805114746094, 'logps/rejected': -1487.4290771484375, 'logps/ref_chosen': -36.464019775390625, 'logps/ref_rejected': -113.0091781616211, 'logits/chosen': -3.458996057510376, 'logits/rejected': -4.9003472328186035, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0008652299293316901, 'epsilon_dpo/loss_margin_mean': 992.203369140625, 'epsilon_dpo/beta_margin_mean': 0.8559463620185852, 'epsilon_dpo/beta_margin_std': 0.6566179990768433, 'epsilon_dpo/beta_margin_grad_mean': -0.3142598271369934, 'epsilon_dpo/beta_margin_grad_std': 0.12845391035079956, 'kl/beta': 0.0008716810261830688, 'kl/avg_steps': 0.75, 'epoch': 0.9}
|
|||
|
|
90%|██████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 613/681 [43:07<04:24, 3.90s/it]
90%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 614/681 [43:10<03:54, 3.51s/it]
{'loss': 0.8482, 'grad_norm': 48.429771423339844, 'learning_rate': 1.507684480352292e-08, 'rewards/chosen': -0.3362823724746704, 'rewards/rejected': -1.1700525283813477, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.8337700366973877, 'logps/chosen': -424.6036376953125, 'logps/rejected': -1475.20361328125, 'logps/ref_chosen': -34.81976318359375, 'logps/ref_rejected': -111.12577819824219, 'logits/chosen': -3.3637969493865967, 'logits/rejected': -4.854522705078125, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0008593298261985183, 'epsilon_dpo/loss_margin_mean': 974.2940063476562, 'epsilon_dpo/beta_margin_mean': 0.8337700366973877, 'epsilon_dpo/beta_margin_std': 0.7923649549484253, 'epsilon_dpo/beta_margin_grad_mean': -0.3246811330318451, 'epsilon_dpo/beta_margin_grad_std': 0.15554648637771606, 'kl/beta': 0.0008651920943520963, 'kl/avg_steps': 0.6875, 'epoch': 0.9}
|
|||
|
|
90%|██████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 614/681 [43:10<03:54, 3.51s/it]
90%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 615/681 [43:12<03:34, 3.25s/it]
{'loss': 0.8908, 'grad_norm': 52.556758880615234, 'learning_rate': 1.4641017128809801e-08, 'rewards/chosen': -0.3474053144454956, 'rewards/rejected': -1.126778483390808, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.7793731689453125, 'logps/chosen': -447.3005676269531, 'logps/rejected': -1424.577392578125, 'logps/ref_chosen': -41.42036819458008, 'logps/ref_rejected': -101.49702453613281, 'logits/chosen': -3.465533971786499, 'logits/rejected': -4.829683303833008, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0008531937492080033, 'epsilon_dpo/loss_margin_mean': 917.2001342773438, 'epsilon_dpo/beta_margin_mean': 0.7793731689453125, 'epsilon_dpo/beta_margin_std': 0.8427423238754272, 'epsilon_dpo/beta_margin_grad_mean': -0.3395027816295624, 'epsilon_dpo/beta_margin_grad_std': 0.15511348843574524, 'kl/beta': 0.0008592845406383276, 'kl/avg_steps': 0.71875, 'epoch': 0.9}
|
|||
|
|
90%|██████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 615/681 [43:12<03:34, 3.25s/it]
90%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 616/681 [43:15<03:18, 3.05s/it]
{'loss': 1.0265, 'grad_norm': 59.032936096191406, 'learning_rate': 1.4211391382180637e-08, 'rewards/chosen': -0.3976947069168091, 'rewards/rejected': -0.9945331811904907, 'rewards/accuracies': 0.828125, 'rewards/margins': 0.5968384146690369, 'logps/chosen': -511.3292236328125, 'logps/rejected': -1254.6151123046875, 'logps/ref_chosen': -45.615753173828125, 'logps/ref_rejected': -80.37959289550781, 'logits/chosen': -3.490535259246826, 'logits/rejected': -4.813992500305176, 'kl/p_epsilon_steps': 0.75, 'kl/n_epsilon_steps': 0.25, 'epsilon_dpo/beta': 0.000848971598315984, 'epsilon_dpo/loss_margin_mean': 708.5220336914062, 'epsilon_dpo/beta_margin_mean': 0.5968384146690369, 'epsilon_dpo/beta_margin_std': 0.8491699695587158, 'epsilon_dpo/beta_margin_grad_mean': -0.3760168254375458, 'epsilon_dpo/beta_margin_grad_std': 0.1653953492641449, 'kl/beta': 0.0008531524799764156, 'kl/avg_steps': 0.5, 'epoch': 0.9}
|
|||
|
|
90%|███████████████████████████████████████████████████████████████████████████████████████████████████████ | 616/681 [43:15<03:18, 3.05s/it]
91%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 617/681 [43:17<03:07, 2.93s/it]
{'loss': 1.088, 'grad_norm': 69.23668670654297, 'learning_rate': 1.378797888467345e-08, 'rewards/chosen': -0.4543150067329407, 'rewards/rejected': -0.927322268486023, 'rewards/accuracies': 0.78125, 'rewards/margins': 0.4730072617530823, 'logps/chosen': -583.8492431640625, 'logps/rejected': -1167.814208984375, 'logps/ref_chosen': -50.210060119628906, 'logps/ref_rejected': -69.55174255371094, 'logits/chosen': -3.5469822883605957, 'logits/rejected': -4.843328475952148, 'kl/p_epsilon_steps': 0.71875, 'kl/n_epsilon_steps': 0.28125, 'epsilon_dpo/beta': 0.0008452784968540072, 'epsilon_dpo/loss_margin_mean': 564.623291015625, 'epsilon_dpo/beta_margin_mean': 0.4730072617530823, 'epsilon_dpo/beta_margin_std': 0.7559821605682373, 'epsilon_dpo/beta_margin_grad_mean': -0.3991197347640991, 'epsilon_dpo/beta_margin_grad_std': 0.14801205694675446, 'kl/beta': 0.0008489079191349447, 'kl/avg_steps': 0.4375, 'epoch': 0.91}
|
|||
|
|
91%|███████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 617/681 [43:18<03:07, 2.93s/it]
91%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 618/681 [43:20<03:02, 2.89s/it]
{'loss': 0.9866, 'grad_norm': 48.00564193725586, 'learning_rate': 1.3370790793601371e-08, 'rewards/chosen': -0.41405919194221497, 'rewards/rejected': -1.0804648399353027, 'rewards/accuracies': 0.875, 'rewards/margins': 0.6664056181907654, 'logps/chosen': -533.0503540039062, 'logps/rejected': -1387.2470703125, 'logps/ref_chosen': -43.185306549072266, 'logps/ref_rejected': -98.49762725830078, 'logits/chosen': -3.4216156005859375, 'logits/rejected': -4.760526657104492, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0008396140765398741, 'epsilon_dpo/loss_margin_mean': 798.8843383789062, 'epsilon_dpo/beta_margin_mean': 0.6664056181907654, 'epsilon_dpo/beta_margin_std': 0.8637328147888184, 'epsilon_dpo/beta_margin_grad_mean': -0.35606545209884644, 'epsilon_dpo/beta_margin_grad_std': 0.17117975652217865, 'kl/beta': 0.0008452101610600948, 'kl/avg_steps': 0.671875, 'epoch': 0.91}
|
|||
|
|
91%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 618/681 [43:20<03:02, 2.89s/it]
91%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 619/681 [43:23<02:54, 2.82s/it]
{'loss': 0.891, 'grad_norm': 55.89384460449219, 'learning_rate': 1.2959838102258535e-08, 'rewards/chosen': -0.37517303228378296, 'rewards/rejected': -1.1234097480773926, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.7482366561889648, 'logps/chosen': -482.2840576171875, 'logps/rejected': -1450.6636962890625, 'logps/ref_chosen': -33.69963836669922, 'logps/ref_rejected': -101.2354736328125, 'logits/chosen': -3.394374370574951, 'logits/rejected': -4.941725254058838, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0008336182800121605, 'epsilon_dpo/loss_margin_mean': 900.8438110351562, 'epsilon_dpo/beta_margin_mean': 0.7482366561889648, 'epsilon_dpo/beta_margin_std': 0.7654976844787598, 'epsilon_dpo/beta_margin_grad_mean': -0.34246736764907837, 'epsilon_dpo/beta_margin_grad_std': 0.14548242092132568, 'kl/beta': 0.0008395693148486316, 'kl/avg_steps': 0.71875, 'epoch': 0.91}
|
|||
|
|
91%|███████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 619/681 [43:23<02:54, 2.82s/it]
91%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 620/681 [43:25<02:44, 2.70s/it]
{'loss': 1.0273, 'grad_norm': 51.175804138183594, 'learning_rate': 1.2555131639630567e-08, 'rewards/chosen': -0.4475584030151367, 'rewards/rejected': -1.0251778364181519, 'rewards/accuracies': 0.796875, 'rewards/margins': 0.5776193737983704, 'logps/chosen': -579.7201538085938, 'logps/rejected': -1323.3944091796875, 'logps/ref_chosen': -42.774513244628906, 'logps/ref_rejected': -84.47439575195312, 'logits/chosen': -3.558187961578369, 'logits/rejected': -4.871739387512207, 'kl/p_epsilon_steps': 0.765625, 'kl/n_epsilon_steps': 0.234375, 'epsilon_dpo/beta': 0.0008292324491776526, 'epsilon_dpo/loss_margin_mean': 701.974365234375, 'epsilon_dpo/beta_margin_mean': 0.5776193737983704, 'epsilon_dpo/beta_margin_std': 0.8028265833854675, 'epsilon_dpo/beta_margin_grad_mean': -0.37484031915664673, 'epsilon_dpo/beta_margin_grad_std': 0.15843062102794647, 'kl/beta': 0.0008335779421031475, 'kl/avg_steps': 0.53125, 'epoch': 0.91}
|
|||
|
|
91%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 620/681 [43:25<02:44, 2.70s/it]
91%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 621/681 [43:28<02:39, 2.66s/it]
{'loss': 0.8891, 'grad_norm': 52.058837890625, 'learning_rate': 1.2156682070109086e-08, 'rewards/chosen': -0.3498554825782776, 'rewards/rejected': -1.1445496082305908, 'rewards/accuracies': 0.828125, 'rewards/margins': 0.794694185256958, 'logps/chosen': -459.840576171875, 'logps/rejected': -1485.9166259765625, 'logps/ref_chosen': -37.82067108154297, 'logps/ref_rejected': -94.49537658691406, 'logits/chosen': -3.53486967086792, 'logits/rejected': -5.024101257324219, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0008243321790359914, 'epsilon_dpo/loss_margin_mean': 969.4013061523438, 'epsilon_dpo/beta_margin_mean': 0.794694185256958, 'epsilon_dpo/beta_margin_std': 0.8520438075065613, 'epsilon_dpo/beta_margin_grad_mean': -0.3368692100048065, 'epsilon_dpo/beta_margin_grad_std': 0.16272993385791779, 'kl/beta': 0.000829172960948199, 'kl/avg_steps': 0.59375, 'epoch': 0.91}
|
|||
|
|
91%|███████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 621/681 [43:28<02:39, 2.66s/it]
91%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 622/681 [43:31<02:37, 2.67s/it]
{'loss': 0.9872, 'grad_norm': 49.092411041259766, 'learning_rate': 1.1764499893210878e-08, 'rewards/chosen': -0.4212498664855957, 'rewards/rejected': -1.0025246143341064, 'rewards/accuracies': 0.875, 'rewards/margins': 0.5812746286392212, 'logps/chosen': -553.1505126953125, 'logps/rejected': -1318.96630859375, 'logps/ref_chosen': -39.961334228515625, 'logps/ref_rejected': -92.28267669677734, 'logits/chosen': -3.374379873275757, 'logits/rejected': -4.7842888832092285, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0008184361504390836, 'epsilon_dpo/loss_margin_mean': 713.4944458007812, 'epsilon_dpo/beta_margin_mean': 0.581274688243866, 'epsilon_dpo/beta_margin_std': 0.6987016797065735, 'epsilon_dpo/beta_margin_grad_mean': -0.3743629455566406, 'epsilon_dpo/beta_margin_grad_std': 0.13544589281082153, 'kl/beta': 0.0008242788026109338, 'kl/avg_steps': 0.71875, 'epoch': 0.91}
|
|||
|
|
91%|████████████████████████████████████████████████████████████████████████████████████████████████████████ | 622/681 [43:31<02:37, 2.67s/it]
91%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 623/681 [43:33<02:29, 2.59s/it]
{'loss': 1.0243, 'grad_norm': 50.80109405517578, 'learning_rate': 1.1378595443300998e-08, 'rewards/chosen': -0.432176798582077, 'rewards/rejected': -1.0211231708526611, 'rewards/accuracies': 0.78125, 'rewards/margins': 0.5889463424682617, 'logps/chosen': -577.177001953125, 'logps/rejected': -1348.3131103515625, 'logps/ref_chosen': -49.0926513671875, 'logps/ref_rejected': -91.09358215332031, 'logits/chosen': -3.542423963546753, 'logits/rejected': -4.91584587097168, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.000813874474260956, 'epsilon_dpo/loss_margin_mean': 729.1351928710938, 'epsilon_dpo/beta_margin_mean': 0.5889464020729065, 'epsilon_dpo/beta_margin_std': 0.8087703585624695, 'epsilon_dpo/beta_margin_grad_mean': -0.3725954592227936, 'epsilon_dpo/beta_margin_grad_std': 0.1656135767698288, 'kl/beta': 0.0008183965692296624, 'kl/avg_steps': 0.5625, 'epoch': 0.91}
|
|||
|
|
91%|████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 623/681 [43:33<02:29, 2.59s/it]
92%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 624/681 [43:36<02:31, 2.66s/it]
{'loss': 0.9153, 'grad_norm': 48.523738861083984, 'learning_rate': 1.0998978889320582e-08, 'rewards/chosen': -0.3814919590950012, 'rewards/rejected': -1.1524710655212402, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.7709789872169495, 'logps/chosen': -515.30517578125, 'logps/rejected': -1532.546142578125, 'logps/ref_chosen': -46.57392501831055, 'logps/ref_rejected': -105.08536529541016, 'logits/chosen': -3.3430838584899902, 'logits/rejected': -4.918627738952637, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.000808813376352191, 'epsilon_dpo/loss_margin_mean': 958.7295532226562, 'epsilon_dpo/beta_margin_mean': 0.7709789872169495, 'epsilon_dpo/beta_margin_std': 0.8832123875617981, 'epsilon_dpo/beta_margin_grad_mean': -0.34141990542411804, 'epsilon_dpo/beta_margin_grad_std': 0.16724152863025665, 'kl/beta': 0.0008138188859447837, 'kl/avg_steps': 0.625, 'epoch': 0.92}
|
|||
|
|
92%|████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 624/681 [43:36<02:31, 2.66s/it]
92%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 625/681 [43:38<02:26, 2.61s/it]
{'loss': 0.902, 'grad_norm': 49.30683135986328, 'learning_rate': 1.0625660234518913e-08, 'rewards/chosen': -0.38019776344299316, 'rewards/rejected': -1.0790600776672363, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.6988623142242432, 'logps/chosen': -515.078857421875, 'logps/rejected': -1437.4361572265625, 'logps/ref_chosen': -43.60509490966797, 'logps/ref_rejected': -92.33833312988281, 'logits/chosen': -3.506286144256592, 'logits/rejected': -4.933239936828613, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0008035369100980461, 'epsilon_dpo/loss_margin_mean': 873.6240844726562, 'epsilon_dpo/beta_margin_mean': 0.6988623738288879, 'epsilon_dpo/beta_margin_std': 0.6729094386100769, 'epsilon_dpo/beta_margin_grad_mean': -0.34749388694763184, 'epsilon_dpo/beta_margin_grad_std': 0.13651293516159058, 'kl/beta': 0.0008087640744633973, 'kl/avg_steps': 0.65625, 'epoch': 0.92}
|
|||
|
|
92%|████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 625/681 [43:38<02:26, 2.61s/it]
92%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 626/681 [43:41<02:24, 2.63s/it]
{'loss': 0.9691, 'grad_norm': 50.068603515625, 'learning_rate': 1.0258649316189721e-08, 'rewards/chosen': -0.43541914224624634, 'rewards/rejected': -1.1092710494995117, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.6738518476486206, 'logps/chosen': -593.0721435546875, 'logps/rejected': -1494.8515625, 'logps/ref_chosen': -50.95122528076172, 'logps/ref_rejected': -103.29271697998047, 'logits/chosen': -3.526883363723755, 'logits/rejected': -4.905704021453857, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0007988003198988736, 'epsilon_dpo/loss_margin_mean': 849.4379272460938, 'epsilon_dpo/beta_margin_mean': 0.6738518476486206, 'epsilon_dpo/beta_margin_std': 0.8334734439849854, 'epsilon_dpo/beta_margin_grad_mean': -0.3574766218662262, 'epsilon_dpo/beta_margin_grad_std': 0.16741888225078583, 'kl/beta': 0.0008034911588765681, 'kl/avg_steps': 0.59375, 'epoch': 0.92}
|
|||
|
|
92%|████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 626/681 [43:41<02:24, 2.63s/it]
92%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 627/681 [43:44<02:21, 2.63s/it]
{'loss': 0.8381, 'grad_norm': 55.83228302001953, 'learning_rate': 9.897955805412e-09, 'rewards/chosen': -0.24587570130825043, 'rewards/rejected': -1.0799684524536133, 'rewards/accuracies': 0.875, 'rewards/margins': 0.8340927362442017, 'logps/chosen': -337.388427734375, 'logps/rejected': -1478.910400390625, 'logps/ref_chosen': -28.80577850341797, 'logps/ref_rejected': -114.96311950683594, 'logits/chosen': -3.3846006393432617, 'logits/rejected': -4.88566780090332, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.00079333659959957, 'epsilon_dpo/loss_margin_mean': 1055.364501953125, 'epsilon_dpo/beta_margin_mean': 0.8340927958488464, 'epsilon_dpo/beta_margin_std': 0.7881202101707458, 'epsilon_dpo/beta_margin_grad_mean': -0.32665178179740906, 'epsilon_dpo/beta_margin_grad_std': 0.14166025817394257, 'kl/beta': 0.0007987486314959824, 'kl/avg_steps': 0.6875, 'epoch': 0.92}
|
|||
|
|
92%|████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 627/681 [43:44<02:21, 2.63s/it]
92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 628/681 [43:46<02:19, 2.63s/it]
{'loss': 0.9161, 'grad_norm': 49.38209915161133, 'learning_rate': 9.543589206795238e-09, 'rewards/chosen': -0.42482954263687134, 'rewards/rejected': -1.1567974090576172, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.7319678068161011, 'logps/chosen': -581.771484375, 'logps/rejected': -1578.183837890625, 'logps/ref_chosen': -45.28186798095703, 'logps/ref_rejected': -108.524169921875, 'logits/chosen': -3.572722911834717, 'logits/rejected': -4.90597677230835, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0007881674682721496, 'epsilon_dpo/loss_margin_mean': 933.1701049804688, 'epsilon_dpo/beta_margin_mean': 0.7319678068161011, 'epsilon_dpo/beta_margin_std': 0.8163183331489563, 'epsilon_dpo/beta_margin_grad_mean': -0.34786510467529297, 'epsilon_dpo/beta_margin_grad_std': 0.15347820520401, 'kl/beta': 0.0007932946900837123, 'kl/avg_steps': 0.65625, 'epoch': 0.92}
|
|||
|
|
92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 628/681 [43:46<02:19, 2.63s/it]
92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 629/681 [43:49<02:18, 2.66s/it]
{'loss': 0.9092, 'grad_norm': 46.65474319458008, 'learning_rate': 9.19555885822887e-09, 'rewards/chosen': -0.3272828459739685, 'rewards/rejected': -1.0314680337905884, 'rewards/accuracies': 0.796875, 'rewards/margins': 0.7041851282119751, 'logps/chosen': -457.57720947265625, 'logps/rejected': -1415.8564453125, 'logps/ref_chosen': -41.636070251464844, 'logps/ref_rejected': -96.60995483398438, 'logits/chosen': -3.379412889480591, 'logits/rejected': -4.922481536865234, 'kl/p_epsilon_steps': 0.765625, 'kl/n_epsilon_steps': 0.234375, 'epsilon_dpo/beta': 0.00078401411883533, 'epsilon_dpo/loss_margin_mean': 903.3053588867188, 'epsilon_dpo/beta_margin_mean': 0.7041851878166199, 'epsilon_dpo/beta_margin_std': 0.7174273133277893, 'epsilon_dpo/beta_margin_grad_mean': -0.34917938709259033, 'epsilon_dpo/beta_margin_grad_std': 0.14244189858436584, 'kl/beta': 0.0007881226483732462, 'kl/avg_steps': 0.53125, 'epoch': 0.92}
|
|||
|
|
92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 629/681 [43:49<02:18, 2.66s/it]
93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 630/681 [43:52<02:15, 2.65s/it]
{'loss': 0.9169, 'grad_norm': 40.582828521728516, 'learning_rate': 8.85387393063622e-09, 'rewards/chosen': -0.26877981424331665, 'rewards/rejected': -0.9388737678527832, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.6700939536094666, 'logps/chosen': -374.94354248046875, 'logps/rejected': -1298.19873046875, 'logps/ref_chosen': -31.366878509521484, 'logps/ref_rejected': -90.5899658203125, 'logits/chosen': -3.2850284576416016, 'logits/rejected': -4.819733619689941, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0007786460919305682, 'epsilon_dpo/loss_margin_mean': 864.0321655273438, 'epsilon_dpo/beta_margin_mean': 0.6700939536094666, 'epsilon_dpo/beta_margin_std': 0.6649956703186035, 'epsilon_dpo/beta_margin_grad_mean': -0.3540729284286499, 'epsilon_dpo/beta_margin_grad_std': 0.13038687407970428, 'kl/beta': 0.0007839578902348876, 'kl/avg_steps': 0.6875, 'epoch': 0.93}
|
|||
|
|
93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 630/681 [43:52<02:15, 2.65s/it]
93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 631/681 [43:54<02:11, 2.63s/it]
{'loss': 0.9527, 'grad_norm': 46.966888427734375, 'learning_rate': 8.518543427732949e-09, 'rewards/chosen': -0.34111732244491577, 'rewards/rejected': -0.9839361906051636, 'rewards/accuracies': 0.875, 'rewards/margins': 0.642818808555603, 'logps/chosen': -483.23895263671875, 'logps/rejected': -1360.65087890625, 'logps/ref_chosen': -44.379119873046875, 'logps/ref_rejected': -86.64693450927734, 'logits/chosen': -3.4160025119781494, 'logits/rejected': -4.869636535644531, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0007730860379524529, 'epsilon_dpo/loss_margin_mean': 835.1441040039062, 'epsilon_dpo/beta_margin_mean': 0.6428188681602478, 'epsilon_dpo/beta_margin_std': 0.7184677124023438, 'epsilon_dpo/beta_margin_grad_mean': -0.3612503111362457, 'epsilon_dpo/beta_margin_grad_std': 0.14340829849243164, 'kl/beta': 0.0007786049391143024, 'kl/avg_steps': 0.71875, 'epoch': 0.93}
|
|||
|
|
93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 631/681 [43:54<02:11, 2.63s/it]
93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 632/681 [43:57<02:04, 2.54s/it]
{'loss': 0.9639, 'grad_norm': 54.08295440673828, 'learning_rate': 8.189576185789637e-09, 'rewards/chosen': -0.35203373432159424, 'rewards/rejected': -0.9687473773956299, 'rewards/accuracies': 0.875, 'rewards/margins': 0.6167136430740356, 'logps/chosen': -500.601318359375, 'logps/rejected': -1354.220947265625, 'logps/ref_chosen': -43.92643737792969, 'logps/ref_rejected': -90.67631530761719, 'logits/chosen': -3.6578102111816406, 'logits/rejected': -5.017969131469727, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0007680523558519781, 'epsilon_dpo/loss_margin_mean': 806.8697509765625, 'epsilon_dpo/beta_margin_mean': 0.6167137026786804, 'epsilon_dpo/beta_margin_std': 0.6938005685806274, 'epsilon_dpo/beta_margin_grad_mean': -0.3662717640399933, 'epsilon_dpo/beta_margin_grad_std': 0.13854503631591797, 'kl/beta': 0.0007730486686341465, 'kl/avg_steps': 0.65625, 'epoch': 0.93}
|
|||
|
|
93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 632/681 [43:57<02:04, 2.54s/it]
93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 633/681 [43:59<02:00, 2.52s/it]
{'loss': 0.9689, 'grad_norm': 49.735836029052734, 'learning_rate': 7.866980873399015e-09, 'rewards/chosen': -0.3694680333137512, 'rewards/rejected': -0.9572535157203674, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.5877854824066162, 'logps/chosen': -524.2335205078125, 'logps/rejected': -1356.517822265625, 'logps/ref_chosen': -42.23455047607422, 'logps/ref_rejected': -100.14579772949219, 'logits/chosen': -3.5298781394958496, 'logits/rejected': -4.785070419311523, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0007630449254065752, 'epsilon_dpo/loss_margin_mean': 774.373046875, 'epsilon_dpo/beta_margin_mean': 0.5877854824066162, 'epsilon_dpo/beta_margin_std': 0.6214975118637085, 'epsilon_dpo/beta_margin_grad_mean': -0.36804521083831787, 'epsilon_dpo/beta_margin_grad_std': 0.1329408884048462, 'kl/beta': 0.000768008641898632, 'kl/avg_steps': 0.65625, 'epoch': 0.93}
|
|||
|
|
93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 633/681 [43:59<02:00, 2.52s/it]
93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 634/681 [44:02<01:59, 2.55s/it]
{'loss': 0.933, 'grad_norm': 43.45764923095703, 'learning_rate': 7.550765991247654e-09, 'rewards/chosen': -0.35321560502052307, 'rewards/rejected': -1.0450574159622192, 'rewards/accuracies': 0.796875, 'rewards/margins': 0.6918417811393738, 'logps/chosen': -502.7445068359375, 'logps/rejected': -1493.4161376953125, 'logps/ref_chosen': -39.36439895629883, 'logps/ref_rejected': -113.15769958496094, 'logits/chosen': -3.4708662033081055, 'logits/rejected': -4.9770402908325195, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.0007587854051962495, 'epsilon_dpo/loss_margin_mean': 916.8782958984375, 'epsilon_dpo/beta_margin_mean': 0.6918417811393738, 'epsilon_dpo/beta_margin_std': 0.782641589641571, 'epsilon_dpo/beta_margin_grad_mean': -0.3558174967765808, 'epsilon_dpo/beta_margin_grad_std': 0.14750294387340546, 'kl/beta': 0.0007630014442838728, 'kl/avg_steps': 0.5625, 'epoch': 0.93}
|
|||
|
|
93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 634/681 [44:02<01:59, 2.55s/it]
93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 635/681 [44:04<01:58, 2.58s/it]
{'loss': 1.0411, 'grad_norm': 51.91278839111328, 'learning_rate': 7.240939871891699e-09, 'rewards/chosen': -0.38917070627212524, 'rewards/rejected': -0.8954064846038818, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.5062357783317566, 'logps/chosen': -562.4381103515625, 'logps/rejected': -1277.62646484375, 'logps/ref_chosen': -49.88642120361328, 'logps/ref_rejected': -89.69390869140625, 'logits/chosen': -3.523716926574707, 'logits/rejected': -4.783617973327637, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0007538297213613987, 'epsilon_dpo/loss_margin_mean': 675.380859375, 'epsilon_dpo/beta_margin_mean': 0.5062357783317566, 'epsilon_dpo/beta_margin_std': 0.6855813264846802, 'epsilon_dpo/beta_margin_grad_mean': -0.39024749398231506, 'epsilon_dpo/beta_margin_grad_std': 0.13376963138580322, 'kl/beta': 0.0007587335421703756, 'kl/avg_steps': 0.65625, 'epoch': 0.93}
|
|||
|
|
93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 635/681 [44:04<01:58, 2.58s/it]
93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 636/681 [44:07<01:59, 2.64s/it]
{'loss': 1.043, 'grad_norm': 52.333683013916016, 'learning_rate': 6.937510679537628e-09, 'rewards/chosen': -0.39245936274528503, 'rewards/rejected': -0.8954870104789734, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.5030276775360107, 'logps/chosen': -566.8447875976562, 'logps/rejected': -1282.334716796875, 'logps/ref_chosen': -46.58656692504883, 'logps/ref_rejected': -86.21536254882812, 'logits/chosen': -3.715489387512207, 'logits/rejected': -4.909304618835449, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0007493861485272646, 'epsilon_dpo/loss_margin_mean': 675.861083984375, 'epsilon_dpo/beta_margin_mean': 0.5030276775360107, 'epsilon_dpo/beta_margin_std': 0.6626613140106201, 'epsilon_dpo/beta_margin_grad_mean': -0.3881148397922516, 'epsilon_dpo/beta_margin_grad_std': 0.1402616947889328, 'kl/beta': 0.0007537868223153055, 'kl/avg_steps': 0.59375, 'epoch': 0.93}
|
|||
|
|
93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 636/681 [44:07<01:59, 2.64s/it]
94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 637/681 [44:10<01:55, 2.64s/it]
{'loss': 0.9332, 'grad_norm': 41.60480880737305, 'learning_rate': 6.640486409826785e-09, 'rewards/chosen': -0.33350521326065063, 'rewards/rejected': -1.0211637020111084, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.6876585483551025, 'logps/chosen': -483.88134765625, 'logps/rejected': -1478.459228515625, 'logps/ref_chosen': -37.54460144042969, 'logps/ref_rejected': -103.94780731201172, 'logits/chosen': -3.4943830966949463, 'logits/rejected': -4.821347236633301, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0007449628901667893, 'epsilon_dpo/loss_margin_mean': 928.1746215820312, 'epsilon_dpo/beta_margin_mean': 0.6876585483551025, 'epsilon_dpo/beta_margin_std': 0.7672451138496399, 'epsilon_dpo/beta_margin_grad_mean': -0.3551463186740875, 'epsilon_dpo/beta_margin_grad_std': 0.14750678837299347, 'kl/beta': 0.0007493376033380628, 'kl/avg_steps': 0.59375, 'epoch': 0.94}
|
|||
|
|
94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 637/681 [44:10<01:55, 2.64s/it]
94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 638/681 [44:12<01:51, 2.59s/it]
{'loss': 0.9433, 'grad_norm': 38.22664260864258, 'learning_rate': 6.349874889624962e-09, 'rewards/chosen': -0.30027681589126587, 'rewards/rejected': -0.9476705193519592, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.6473937034606934, 'logps/chosen': -439.9283447265625, 'logps/rejected': -1368.220947265625, 'logps/ref_chosen': -35.51661682128906, 'logps/ref_rejected': -85.09121704101562, 'logits/chosen': -3.2752623558044434, 'logits/rejected': -5.041168212890625, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0007394017884507775, 'epsilon_dpo/loss_margin_mean': 878.7179565429688, 'epsilon_dpo/beta_margin_mean': 0.6473937034606934, 'epsilon_dpo/beta_margin_std': 0.7178838849067688, 'epsilon_dpo/beta_margin_grad_mean': -0.3611353039741516, 'epsilon_dpo/beta_margin_grad_std': 0.13404038548469543, 'kl/beta': 0.0007449146942235529, 'kl/avg_steps': 0.75, 'epoch': 0.94}
|
|||
|
|
94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 638/681 [44:12<01:51, 2.59s/it]
94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 639/681 [44:15<01:48, 2.60s/it]
{'loss': 1.0788, 'grad_norm': 46.616573333740234, 'learning_rate': 6.065683776815933e-09, 'rewards/chosen': -0.3750152587890625, 'rewards/rejected': -0.8208526372909546, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.4458373486995697, 'logps/chosen': -551.689208984375, 'logps/rejected': -1200.0047607421875, 'logps/ref_chosen': -44.109619140625, 'logps/ref_rejected': -81.57601928710938, 'logits/chosen': -3.443708896636963, 'logits/rejected': -4.675836563110352, 'kl/p_epsilon_steps': 0.765625, 'kl/n_epsilon_steps': 0.234375, 'epsilon_dpo/beta': 0.0007355150883086026, 'epsilon_dpo/loss_margin_mean': 610.8491821289062, 'epsilon_dpo/beta_margin_mean': 0.4458373486995697, 'epsilon_dpo/beta_margin_std': 0.6368516087532043, 'epsilon_dpo/beta_margin_grad_mean': -0.400001585483551, 'epsilon_dpo/beta_margin_grad_std': 0.13325192034244537, 'kl/beta': 0.0007393694249913096, 'kl/avg_steps': 0.53125, 'epoch': 0.94}
|
|||
|
|
94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 639/681 [44:15<01:48, 2.60s/it]
94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 640/681 [44:17<01:45, 2.58s/it]
{'loss': 0.8862, 'grad_norm': 47.58893966674805, 'learning_rate': 5.7879205600998296e-09, 'rewards/chosen': -0.2974758744239807, 'rewards/rejected': -1.0234477519989014, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.7259718179702759, 'logps/chosen': -445.7485656738281, 'logps/rejected': -1517.7373046875, 'logps/ref_chosen': -40.14595413208008, 'logps/ref_rejected': -114.68016815185547, 'logits/chosen': -3.3290483951568604, 'logits/rejected': -4.9194793701171875, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0007302492158487439, 'epsilon_dpo/loss_margin_mean': 997.45458984375, 'epsilon_dpo/beta_margin_mean': 0.7259718775749207, 'epsilon_dpo/beta_margin_std': 0.7215431928634644, 'epsilon_dpo/beta_margin_grad_mean': -0.34541165828704834, 'epsilon_dpo/beta_margin_grad_std': 0.12778370082378387, 'kl/beta': 0.0007354622939601541, 'kl/avg_steps': 0.71875, 'epoch': 0.94}
|
|||
|
|
94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 640/681 [44:17<01:45, 2.58s/it]
94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 641/681 [44:20<01:43, 2.59s/it]
{'loss': 0.9185, 'grad_norm': 43.9137077331543, 'learning_rate': 5.516592558795746e-09, 'rewards/chosen': -0.3105979263782501, 'rewards/rejected': -1.0035767555236816, 'rewards/accuracies': 0.875, 'rewards/margins': 0.6929788589477539, 'logps/chosen': -465.90545654296875, 'logps/rejected': -1480.5869140625, 'logps/ref_chosen': -39.17839050292969, 'logps/ref_rejected': -94.15284729003906, 'logits/chosen': -3.348452568054199, 'logits/rejected': -5.015802383422852, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0007254944066517055, 'epsilon_dpo/loss_margin_mean': 959.70703125, 'epsilon_dpo/beta_margin_mean': 0.6929788589477539, 'epsilon_dpo/beta_margin_std': 0.7391694188117981, 'epsilon_dpo/beta_margin_grad_mean': -0.3516106605529785, 'epsilon_dpo/beta_margin_grad_std': 0.14073027670383453, 'kl/beta': 0.0007302138837985694, 'kl/avg_steps': 0.65625, 'epoch': 0.94}
|
|||
|
|
94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 641/681 [44:20<01:43, 2.59s/it]
94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 642/681 [44:23<01:42, 2.62s/it]
{'loss': 0.8997, 'grad_norm': 42.03846740722656, 'learning_rate': 5.251706922648868e-09, 'rewards/chosen': -0.3134210705757141, 'rewards/rejected': -1.0650629997253418, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.7516418695449829, 'logps/chosen': -480.06494140625, 'logps/rejected': -1596.8970947265625, 'logps/ref_chosen': -46.66090393066406, 'logps/ref_rejected': -115.78807067871094, 'logits/chosen': -3.4749832153320312, 'logits/rejected': -5.020747184753418, 'kl/p_epsilon_steps': 0.875, 'kl/n_epsilon_steps': 0.125, 'epsilon_dpo/beta': 0.0007200841791927814, 'epsilon_dpo/loss_margin_mean': 1047.7049560546875, 'epsilon_dpo/beta_margin_mean': 0.7516418695449829, 'epsilon_dpo/beta_margin_std': 0.7943530082702637, 'epsilon_dpo/beta_margin_grad_mean': -0.34033486247062683, 'epsilon_dpo/beta_margin_grad_std': 0.1520089954137802, 'kl/beta': 0.0007254530792124569, 'kl/avg_steps': 0.75, 'epoch': 0.94}
|
|||
|
|
94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 642/681 [44:23<01:42, 2.62s/it]
94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 643/681 [44:25<01:37, 2.57s/it]
{'loss': 0.9222, 'grad_norm': 39.312984466552734, 'learning_rate': 4.993270631642038e-09, 'rewards/chosen': -0.29885566234588623, 'rewards/rejected': -0.9357801079750061, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.6369244456291199, 'logps/chosen': -453.1264343261719, 'logps/rejected': -1403.9027099609375, 'logps/ref_chosen': -35.49954605102539, 'logps/ref_rejected': -92.612060546875, 'logits/chosen': -3.4138784408569336, 'logits/rejected': -4.991301536560059, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0007149488083086908, 'epsilon_dpo/loss_margin_mean': 893.6637573242188, 'epsilon_dpo/beta_margin_mean': 0.6369244456291199, 'epsilon_dpo/beta_margin_std': 0.594021201133728, 'epsilon_dpo/beta_margin_grad_mean': -0.3584926128387451, 'epsilon_dpo/beta_margin_grad_std': 0.11874385923147202, 'kl/beta': 0.0007200526888482273, 'kl/avg_steps': 0.71875, 'epoch': 0.94}
|
|||
|
|
94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 643/681 [44:25<01:37, 2.57s/it]
95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 644/681 [44:28<01:37, 2.63s/it]
{'loss': 0.9704, 'grad_norm': 46.13814926147461, 'learning_rate': 4.741290495811873e-09, 'rewards/chosen': -0.28092193603515625, 'rewards/rejected': -0.8519556522369385, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.5710337162017822, 'logps/chosen': -432.34515380859375, 'logps/rejected': -1295.53857421875, 'logps/ref_chosen': -37.63022232055664, 'logps/ref_rejected': -93.44629669189453, 'logits/chosen': -3.398604154586792, 'logits/rejected': -4.7896952629089355, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0007100701914168894, 'epsilon_dpo/loss_margin_mean': 807.3773193359375, 'epsilon_dpo/beta_margin_mean': 0.5710337162017822, 'epsilon_dpo/beta_margin_std': 0.5858953595161438, 'epsilon_dpo/beta_margin_grad_mean': -0.37228909134864807, 'epsilon_dpo/beta_margin_grad_std': 0.1232423335313797, 'kl/beta': 0.0007149142329581082, 'kl/avg_steps': 0.6875, 'epoch': 0.95}
|
|||
|
|
95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 644/681 [44:28<01:37, 2.63s/it]
95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 645/681 [44:31<01:35, 2.65s/it]
{'loss': 1.0346, 'grad_norm': 42.622047424316406, 'learning_rate': 4.495773155069299e-09, 'rewards/chosen': -0.35279667377471924, 'rewards/rejected': -0.8792734146118164, 'rewards/accuracies': 0.828125, 'rewards/margins': 0.5264767408370972, 'logps/chosen': -535.257568359375, 'logps/rejected': -1353.5076904296875, 'logps/ref_chosen': -37.85113525390625, 'logps/ref_rejected': -105.40227508544922, 'logits/chosen': -3.5165293216705322, 'logits/rejected': -4.699129104614258, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.0007061094511300325, 'epsilon_dpo/loss_margin_mean': 750.698974609375, 'epsilon_dpo/beta_margin_mean': 0.5264767408370972, 'epsilon_dpo/beta_margin_std': 0.6991389393806458, 'epsilon_dpo/beta_margin_grad_mean': -0.38581857085227966, 'epsilon_dpo/beta_margin_grad_std': 0.14517748355865479, 'kl/beta': 0.0007100327638909221, 'kl/avg_steps': 0.5625, 'epoch': 0.95}
|
|||
|
|
95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 645/681 [44:31<01:35, 2.65s/it]
95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 646/681 [44:33<01:32, 2.63s/it]
{'loss': 1.0461, 'grad_norm': 47.977237701416016, 'learning_rate': 4.256725079024553e-09, 'rewards/chosen': -0.33754080533981323, 'rewards/rejected': -0.8128384947776794, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.4752976894378662, 'logps/chosen': -520.402099609375, 'logps/rejected': -1243.4267578125, 'logps/ref_chosen': -41.30128860473633, 'logps/ref_rejected': -82.82234954833984, 'logits/chosen': -3.479154109954834, 'logits/rejected': -4.7768402099609375, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0007019391632638872, 'epsilon_dpo/loss_margin_mean': 681.503662109375, 'epsilon_dpo/beta_margin_mean': 0.4752976894378662, 'epsilon_dpo/beta_margin_std': 0.5920587778091431, 'epsilon_dpo/beta_margin_grad_mean': -0.39241456985473633, 'epsilon_dpo/beta_margin_grad_std': 0.129283145070076, 'kl/beta': 0.0007060611969791353, 'kl/avg_steps': 0.59375, 'epoch': 0.95}
|
|||
|
|
95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 646/681 [44:33<01:32, 2.63s/it]
95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 647/681 [44:36<01:30, 2.67s/it]
{'loss': 0.9654, 'grad_norm': 48.550132751464844, 'learning_rate': 4.024152566816791e-09, 'rewards/chosen': -0.2764018774032593, 'rewards/rejected': -0.8342074155807495, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.5578055381774902, 'logps/chosen': -431.2979736328125, 'logps/rejected': -1297.52197265625, 'logps/ref_chosen': -35.967567443847656, 'logps/ref_rejected': -98.74945068359375, 'logits/chosen': -3.453265905380249, 'logits/rejected': -4.754009246826172, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0006964798085391521, 'epsilon_dpo/loss_margin_mean': 803.4420776367188, 'epsilon_dpo/beta_margin_mean': 0.5578054785728455, 'epsilon_dpo/beta_margin_std': 0.5175791382789612, 'epsilon_dpo/beta_margin_grad_mean': -0.3713095486164093, 'epsilon_dpo/beta_margin_grad_std': 0.11274945735931396, 'kl/beta': 0.0007018937030807137, 'kl/avg_steps': 0.78125, 'epoch': 0.95}
|
|||
|
|
95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 647/681 [44:36<01:30, 2.67s/it]
95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 648/681 [44:38<01:27, 2.64s/it]
{'loss': 0.9453, 'grad_norm': 52.17890930175781, 'learning_rate': 3.798061746947995e-09, 'rewards/chosen': -0.2765774726867676, 'rewards/rejected': -0.927411675453186, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.6508342027664185, 'logps/chosen': -432.31951904296875, 'logps/rejected': -1449.33203125, 'logps/ref_chosen': -33.676727294921875, 'logps/ref_rejected': -105.11663818359375, 'logits/chosen': -3.3199658393859863, 'logits/rejected': -4.965579986572266, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0006910806987434626, 'epsilon_dpo/loss_margin_mean': 945.5726318359375, 'epsilon_dpo/beta_margin_mean': 0.6508342027664185, 'epsilon_dpo/beta_margin_std': 0.699810802936554, 'epsilon_dpo/beta_margin_grad_mean': -0.35618698596954346, 'epsilon_dpo/beta_margin_grad_std': 0.14514584839344025, 'kl/beta': 0.0006964526255615056, 'kl/avg_steps': 0.78125, 'epoch': 0.95}
|
|||
|
|
95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 648/681 [44:38<01:27, 2.64s/it]
95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 649/681 [44:41<01:25, 2.67s/it]
{'loss': 1.1091, 'grad_norm': 40.87678146362305, 'learning_rate': 3.5784585771215235e-09, 'rewards/chosen': -0.3326740860939026, 'rewards/rejected': -0.7718292474746704, 'rewards/accuracies': 0.78125, 'rewards/margins': 0.43915510177612305, 'logps/chosen': -526.4951171875, 'logps/rejected': -1212.298095703125, 'logps/ref_chosen': -45.06011199951172, 'logps/ref_rejected': -86.40021514892578, 'logits/chosen': -3.504033088684082, 'logits/rejected': -4.718470573425293, 'kl/p_epsilon_steps': 0.765625, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.0006873422535136342, 'epsilon_dpo/loss_margin_mean': 644.4627685546875, 'epsilon_dpo/beta_margin_mean': 0.43915510177612305, 'epsilon_dpo/beta_margin_std': 0.7278276085853577, 'epsilon_dpo/beta_margin_grad_mean': -0.40578776597976685, 'epsilon_dpo/beta_margin_grad_std': 0.1477607786655426, 'kl/beta': 0.0006910538068041205, 'kl/avg_steps': 0.546875, 'epoch': 0.95}
|
|||
|
|
95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 649/681 [44:41<01:25, 2.67s/it]
95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 650/681 [44:44<01:20, 2.61s/it]
{'loss': 0.957, 'grad_norm': 45.862091064453125, 'learning_rate': 3.3653488440851253e-09, 'rewards/chosen': -0.36932051181793213, 'rewards/rejected': -1.0521395206451416, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.682819128036499, 'logps/chosen': -577.81494140625, 'logps/rejected': -1647.8070068359375, 'logps/ref_chosen': -39.21210861206055, 'logps/ref_rejected': -103.7435302734375, 'logits/chosen': -3.553635835647583, 'logits/rejected': -5.113847732543945, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.0006830678903497756, 'epsilon_dpo/loss_margin_mean': 1005.4606323242188, 'epsilon_dpo/beta_margin_mean': 0.6828190684318542, 'epsilon_dpo/beta_margin_std': 0.8265655040740967, 'epsilon_dpo/beta_margin_grad_mean': -0.3580322861671448, 'epsilon_dpo/beta_margin_grad_std': 0.16048245131969452, 'kl/beta': 0.0006872951635159552, 'kl/avg_steps': 0.625, 'epoch': 0.95}
|
|||
|
|
95%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 650/681 [44:44<01:20, 2.61s/it]
96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 651/681 [44:46<01:17, 2.58s/it]
{'loss': 0.9624, 'grad_norm': 40.79838943481445, 'learning_rate': 3.158738163478475e-09, 'rewards/chosen': -0.3056422173976898, 'rewards/rejected': -0.9292333126068115, 'rewards/accuracies': 0.828125, 'rewards/margins': 0.6235911250114441, 'logps/chosen': -481.7735595703125, 'logps/rejected': -1477.223876953125, 'logps/ref_chosen': -34.16796875, 'logps/ref_rejected': -105.96416473388672, 'logits/chosen': -3.4843456745147705, 'logits/rejected': -4.997735500335693, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0006790386396460235, 'epsilon_dpo/loss_margin_mean': 923.6541137695312, 'epsilon_dpo/beta_margin_mean': 0.6235911250114441, 'epsilon_dpo/beta_margin_std': 0.7001570463180542, 'epsilon_dpo/beta_margin_grad_mean': -0.3646124601364136, 'epsilon_dpo/beta_margin_grad_std': 0.14354108273983002, 'kl/beta': 0.0006830262136645615, 'kl/avg_steps': 0.59375, 'epoch': 0.96}
|
|||
|
|
96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 651/681 [44:46<01:17, 2.58s/it]
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 652/681 [44:49<01:16, 2.64s/it]
{'loss': 0.9666, 'grad_norm': 47.86579895019531, 'learning_rate': 2.9586319796851555e-09, 'rewards/chosen': -0.32234591245651245, 'rewards/rejected': -0.9603723883628845, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.6380264759063721, 'logps/chosen': -517.8300170898438, 'logps/rejected': -1544.671142578125, 'logps/ref_chosen': -43.1708984375, 'logps/ref_rejected': -119.19691467285156, 'logits/chosen': -3.3652729988098145, 'logits/rejected': -4.884889602661133, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0006750306929461658, 'epsilon_dpo/loss_margin_mean': 950.8151245117188, 'epsilon_dpo/beta_margin_mean': 0.6380264759063721, 'epsilon_dpo/beta_margin_std': 0.7482752203941345, 'epsilon_dpo/beta_margin_grad_mean': -0.36323583126068115, 'epsilon_dpo/beta_margin_grad_std': 0.1515437662601471, 'kl/beta': 0.0006789946928620338, 'kl/avg_steps': 0.59375, 'epoch': 0.96}
|
|||
|
|
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 652/681 [44:49<01:16, 2.64s/it]
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 653/681 [44:52<01:13, 2.63s/it]
{'loss': 0.8559, 'grad_norm': 48.55562973022461, 'learning_rate': 2.7650355656892166e-09, 'rewards/chosen': -0.28941768407821655, 'rewards/rejected': -1.0600758790969849, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.7706582546234131, 'logps/chosen': -467.5439453125, 'logps/rejected': -1696.89404296875, 'logps/ref_chosen': -36.02939987182617, 'logps/ref_rejected': -111.6745376586914, 'logits/chosen': -3.2305068969726562, 'logits/rejected': -5.08480167388916, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0006693587056361139, 'epsilon_dpo/loss_margin_mean': 1153.705078125, 'epsilon_dpo/beta_margin_mean': 0.7706581950187683, 'epsilon_dpo/beta_margin_std': 0.685066819190979, 'epsilon_dpo/beta_margin_grad_mean': -0.3340756595134735, 'epsilon_dpo/beta_margin_grad_std': 0.13510021567344666, 'kl/beta': 0.0006749869789928198, 'kl/avg_steps': 0.84375, 'epoch': 0.96}
|
|||
|
|
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 653/681 [44:52<01:13, 2.63s/it]
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 654/681 [44:54<01:11, 2.63s/it]
{'loss': 0.9534, 'grad_norm': 41.65330505371094, 'learning_rate': 2.577954022936174e-09, 'rewards/chosen': -0.2903170585632324, 'rewards/rejected': -0.8954617977142334, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.605144739151001, 'logps/chosen': -473.72698974609375, 'logps/rejected': -1454.8271484375, 'logps/ref_chosen': -38.32045364379883, 'logps/ref_rejected': -105.32196807861328, 'logits/chosen': -3.4308786392211914, 'logits/rejected': -4.949398040771484, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0006645949906669557, 'epsilon_dpo/loss_margin_mean': 914.0985717773438, 'epsilon_dpo/beta_margin_mean': 0.605144739151001, 'epsilon_dpo/beta_margin_std': 0.6342909932136536, 'epsilon_dpo/beta_margin_grad_mean': -0.36657387018203735, 'epsilon_dpo/beta_margin_grad_std': 0.1234474629163742, 'kl/beta': 0.0006693393806926906, 'kl/avg_steps': 0.71875, 'epoch': 0.96}
|
|||
|
|
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 654/681 [44:54<01:11, 2.63s/it]
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 655/681 [44:57<01:07, 2.61s/it]
{'loss': 0.9735, 'grad_norm': 43.04616165161133, 'learning_rate': 2.397392281198729e-09, 'rewards/chosen': -0.23886506259441376, 'rewards/rejected': -0.8641390800476074, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.6252740025520325, 'logps/chosen': -389.5122985839844, 'logps/rejected': -1416.551025390625, 'logps/ref_chosen': -29.801528930664062, 'logps/ref_rejected': -104.75204467773438, 'logits/chosen': -3.374847173690796, 'logits/rejected': -4.775622844696045, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.000660475343465805, 'epsilon_dpo/loss_margin_mean': 952.0883178710938, 'epsilon_dpo/beta_margin_mean': 0.6252740025520325, 'epsilon_dpo/beta_margin_std': 0.7545697093009949, 'epsilon_dpo/beta_margin_grad_mean': -0.3668808341026306, 'epsilon_dpo/beta_margin_grad_std': 0.14767590165138245, 'kl/beta': 0.0006645628600381315, 'kl/avg_steps': 0.625, 'epoch': 0.96}
|
|||
|
|
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 655/681 [44:57<01:07, 2.61s/it]
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 656/681 [44:59<01:05, 2.63s/it]
{'loss': 0.8158, 'grad_norm': 51.54441833496094, 'learning_rate': 2.223355098446622e-09, 'rewards/chosen': -0.2671954035758972, 'rewards/rejected': -1.0838143825531006, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.8166189193725586, 'logps/chosen': -448.05487060546875, 'logps/rejected': -1777.021484375, 'logps/ref_chosen': -40.732295989990234, 'logps/ref_rejected': -120.36123657226562, 'logits/chosen': -3.3955602645874023, 'logits/rejected': -5.103156089782715, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0006549282115884125, 'epsilon_dpo/loss_margin_mean': 1249.337646484375, 'epsilon_dpo/beta_margin_mean': 0.8166189193725586, 'epsilon_dpo/beta_margin_std': 0.6401075720787048, 'epsilon_dpo/beta_margin_grad_mean': -0.32215529680252075, 'epsilon_dpo/beta_margin_grad_std': 0.1285041719675064, 'kl/beta': 0.0006604351219721138, 'kl/avg_steps': 0.84375, 'epoch': 0.96}
|
|||
|
|
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 656/681 [44:59<01:05, 2.63s/it]
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 657/681 [45:02<01:01, 2.57s/it]
{'loss': 0.9133, 'grad_norm': 43.996578216552734, 'learning_rate': 2.055847060721566e-09, 'rewards/chosen': -0.29575812816619873, 'rewards/rejected': -0.9470652341842651, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.6513071060180664, 'logps/chosen': -486.0547790527344, 'logps/rejected': -1563.21728515625, 'logps/ref_chosen': -32.56511688232422, 'logps/ref_rejected': -104.74242401123047, 'logits/chosen': -3.505584716796875, 'logits/rejected': -5.049525260925293, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0006502671749331057, 'epsilon_dpo/loss_margin_mean': 1004.9851684570312, 'epsilon_dpo/beta_margin_mean': 0.6513071060180664, 'epsilon_dpo/beta_margin_std': 0.5884512662887573, 'epsilon_dpo/beta_margin_grad_mean': -0.3548159599304199, 'epsilon_dpo/beta_margin_grad_std': 0.12233418226242065, 'kl/beta': 0.0006549093523062766, 'kl/avg_steps': 0.71875, 'epoch': 0.96}
|
|||
|
|
96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 657/681 [45:02<01:01, 2.57s/it]
97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 658/681 [45:05<00:59, 2.61s/it]
{'loss': 0.9409, 'grad_norm': 35.85395431518555, 'learning_rate': 1.8948725820160662e-09, 'rewards/chosen': -0.2720116972923279, 'rewards/rejected': -0.8697974681854248, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.5977858304977417, 'logps/chosen': -452.764404296875, 'logps/rejected': -1451.1217041015625, 'logps/ref_chosen': -32.16458511352539, 'logps/ref_rejected': -100.98091125488281, 'logits/chosen': -3.3597793579101562, 'logits/rejected': -5.021791458129883, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0006452203379012644, 'epsilon_dpo/loss_margin_mean': 929.5409545898438, 'epsilon_dpo/beta_margin_mean': 0.5977858304977417, 'epsilon_dpo/beta_margin_std': 0.5393864512443542, 'epsilon_dpo/beta_margin_grad_mean': -0.36330175399780273, 'epsilon_dpo/beta_margin_grad_std': 0.11596217751502991, 'kl/beta': 0.0006502358010038733, 'kl/avg_steps': 0.78125, 'epoch': 0.97}
|
|||
|
|
97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 658/681 [45:05<00:59, 2.61s/it]
97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 659/681 [45:07<00:57, 2.63s/it]
{'loss': 0.9852, 'grad_norm': 40.669891357421875, 'learning_rate': 1.7404359041573723e-09, 'rewards/chosen': -0.2872786521911621, 'rewards/rejected': -0.8520359992980957, 'rewards/accuracies': 0.828125, 'rewards/margins': 0.5647573471069336, 'logps/chosen': -490.76068115234375, 'logps/rejected': -1424.7255859375, 'logps/ref_chosen': -44.455406188964844, 'logps/ref_rejected': -93.29725646972656, 'logits/chosen': -3.3426105976104736, 'logits/rejected': -4.952641487121582, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0006414285162463784, 'epsilon_dpo/loss_margin_mean': 885.123046875, 'epsilon_dpo/beta_margin_mean': 0.5647572875022888, 'epsilon_dpo/beta_margin_std': 0.6336069703102112, 'epsilon_dpo/beta_margin_grad_mean': -0.3757038414478302, 'epsilon_dpo/beta_margin_grad_std': 0.12834706902503967, 'kl/beta': 0.0006451951921917498, 'kl/avg_steps': 0.59375, 'epoch': 0.97}
|
|||
|
|
97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 659/681 [45:07<00:57, 2.63s/it]
97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 660/681 [45:10<00:55, 2.65s/it]
{'loss': 0.9552, 'grad_norm': 37.189697265625, 'learning_rate': 1.592541096695571e-09, 'rewards/chosen': -0.26376718282699585, 'rewards/rejected': -0.8527300953865051, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.5889629125595093, 'logps/chosen': -454.3048095703125, 'logps/rejected': -1423.1195068359375, 'logps/ref_chosen': -41.76560974121094, 'logps/ref_rejected': -82.32925415039062, 'logits/chosen': -3.402561664581299, 'logits/rejected': -5.116570472717285, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0006372415809892118, 'epsilon_dpo/loss_margin_mean': 928.2510375976562, 'epsilon_dpo/beta_margin_mean': 0.5889629125595093, 'epsilon_dpo/beta_margin_std': 0.5796295404434204, 'epsilon_dpo/beta_margin_grad_mean': -0.36822497844696045, 'epsilon_dpo/beta_margin_grad_std': 0.1204964891076088, 'kl/beta': 0.0006413869559764862, 'kl/avg_steps': 0.65625, 'epoch': 0.97}
|
|||
|
|
97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 660/681 [45:10<00:55, 2.65s/it]
97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 661/681 [45:12<00:51, 2.55s/it]
{'loss': 0.9829, 'grad_norm': 32.038639068603516, 'learning_rate': 1.4511920567963908e-09, 'rewards/chosen': -0.29568618535995483, 'rewards/rejected': -0.8556575775146484, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.5599713921546936, 'logps/chosen': -504.87115478515625, 'logps/rejected': -1446.969482421875, 'logps/ref_chosen': -38.271453857421875, 'logps/ref_rejected': -92.10589599609375, 'logits/chosen': -3.5372705459594727, 'logits/rejected': -4.98753547668457, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0006320912507362664, 'epsilon_dpo/loss_margin_mean': 888.2638549804688, 'epsilon_dpo/beta_margin_mean': 0.5599713921546936, 'epsilon_dpo/beta_margin_std': 0.6463156938552856, 'epsilon_dpo/beta_margin_grad_mean': -0.3780907392501831, 'epsilon_dpo/beta_margin_grad_std': 0.11522994190454483, 'kl/beta': 0.0006372053176164627, 'kl/avg_steps': 0.8125, 'epoch': 0.97}
|
|||
|
|
97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 661/681 [45:12<00:51, 2.55s/it]
97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 662/681 [45:15<00:48, 2.54s/it]
{'loss': 0.9818, 'grad_norm': 37.34529113769531, 'learning_rate': 1.3163925091384532e-09, 'rewards/chosen': -0.31171175837516785, 'rewards/rejected': -0.9033301472663879, 'rewards/accuracies': 0.875, 'rewards/margins': 0.5916184186935425, 'logps/chosen': -533.3956298828125, 'logps/rejected': -1537.224365234375, 'logps/ref_chosen': -39.42928695678711, 'logps/ref_rejected': -96.40357208251953, 'logits/chosen': -3.4751718044281006, 'logits/rejected': -5.172747611999512, 'kl/p_epsilon_steps': 0.828125, 'kl/n_epsilon_steps': 0.171875, 'epsilon_dpo/beta': 0.0006279845256358385, 'epsilon_dpo/loss_margin_mean': 946.8544311523438, 'epsilon_dpo/beta_margin_mean': 0.5916183590888977, 'epsilon_dpo/beta_margin_std': 0.681064784526825, 'epsilon_dpo/beta_margin_grad_mean': -0.3694445788860321, 'epsilon_dpo/beta_margin_grad_std': 0.1416751593351364, 'kl/beta': 0.0006320697139017284, 'kl/avg_steps': 0.65625, 'epoch': 0.97}
|
|||
|
|
97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 662/681 [45:15<00:48, 2.54s/it]
97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 663/681 [45:17<00:45, 2.55s/it]
{'loss': 0.9729, 'grad_norm': 46.52898025512695, 'learning_rate': 1.1881460058152382e-09, 'rewards/chosen': -0.2980585992336273, 'rewards/rejected': -0.9053161144256592, 'rewards/accuracies': 0.828125, 'rewards/margins': 0.6072574853897095, 'logps/chosen': -518.152099609375, 'logps/rejected': -1573.593017578125, 'logps/ref_chosen': -44.08625411987305, 'logps/ref_rejected': -121.31452178955078, 'logits/chosen': -3.4985265731811523, 'logits/rejected': -4.843735694885254, 'kl/p_epsilon_steps': 0.765625, 'kl/n_epsilon_steps': 0.234375, 'epsilon_dpo/beta': 0.0006246752454899251, 'epsilon_dpo/loss_margin_mean': 978.2127075195312, 'epsilon_dpo/beta_margin_mean': 0.6072575449943542, 'epsilon_dpo/beta_margin_std': 0.6948506832122803, 'epsilon_dpo/beta_margin_grad_mean': -0.3684435486793518, 'epsilon_dpo/beta_margin_grad_std': 0.14285063743591309, 'kl/beta': 0.0006279487861320376, 'kl/avg_steps': 0.53125, 'epoch': 0.97}
|
|||
|
|
97%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 663/681 [45:17<00:45, 2.55s/it]
98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 664/681 [45:20<00:44, 2.60s/it]
{'loss': 0.8558, 'grad_norm': 43.377994537353516, 'learning_rate': 1.066455926241383e-09, 'rewards/chosen': -0.19074614346027374, 'rewards/rejected': -0.943513035774231, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.7527669072151184, 'logps/chosen': -338.7730712890625, 'logps/rejected': -1640.69140625, 'logps/ref_chosen': -31.542118072509766, 'logps/ref_rejected': -116.06498718261719, 'logits/chosen': -3.1317358016967773, 'logits/rejected': -5.002200126647949, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0006194221205078065, 'epsilon_dpo/loss_margin_mean': 1217.3953857421875, 'epsilon_dpo/beta_margin_mean': 0.7527669072151184, 'epsilon_dpo/beta_margin_std': 0.655570924282074, 'epsilon_dpo/beta_margin_grad_mean': -0.33726438879966736, 'epsilon_dpo/beta_margin_grad_std': 0.1218588799238205, 'kl/beta': 0.0006246304837986827, 'kl/avg_steps': 0.84375, 'epoch': 0.98}
|
|||
|
|
98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 664/681 [45:20<00:44, 2.60s/it]
98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 665/681 [45:23<00:41, 2.58s/it]
{'loss': 0.9663, 'grad_norm': 44.8764762878418, 'learning_rate': 9.513254770636137e-10, 'rewards/chosen': -0.2572126090526581, 'rewards/rejected': -0.8239928483963013, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.5667802095413208, 'logps/chosen': -454.4164123535156, 'logps/rejected': -1432.787353515625, 'logps/ref_chosen': -36.933109283447266, 'logps/ref_rejected': -89.9020767211914, 'logits/chosen': -3.3389735221862793, 'logits/rejected': -5.07022762298584, 'kl/p_epsilon_steps': 0.921875, 'kl/n_epsilon_steps': 0.078125, 'epsilon_dpo/beta': 0.0006142394850030541, 'epsilon_dpo/loss_margin_mean': 925.40185546875, 'epsilon_dpo/beta_margin_mean': 0.5667802691459656, 'epsilon_dpo/beta_margin_std': 0.5540540218353271, 'epsilon_dpo/beta_margin_grad_mean': -0.37084051966667175, 'epsilon_dpo/beta_margin_grad_std': 0.11781810969114304, 'kl/beta': 0.0006194042507559061, 'kl/avg_steps': 0.84375, 'epoch': 0.98}
|
|||
|
|
98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 665/681 [45:23<00:41, 2.58s/it]
98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 666/681 [45:25<00:38, 2.60s/it]
{'loss': 1.0509, 'grad_norm': 41.77029037475586, 'learning_rate': 8.427576920763956e-10, 'rewards/chosen': -0.3538281321525574, 'rewards/rejected': -0.8195853233337402, 'rewards/accuracies': 0.796875, 'rewards/margins': 0.46575719118118286, 'logps/chosen': -623.1746826171875, 'logps/rejected': -1445.141845703125, 'logps/ref_chosen': -47.59907913208008, 'logps/ref_rejected': -102.33778381347656, 'logits/chosen': -3.6731343269348145, 'logits/rejected': -4.908390522003174, 'kl/p_epsilon_steps': 0.765625, 'kl/n_epsilon_steps': 0.234375, 'epsilon_dpo/beta': 0.0006110197864472866, 'epsilon_dpo/loss_margin_mean': 767.228515625, 'epsilon_dpo/beta_margin_mean': 0.46575719118118286, 'epsilon_dpo/beta_margin_std': 0.5870808362960815, 'epsilon_dpo/beta_margin_grad_mean': -0.39552050828933716, 'epsilon_dpo/beta_margin_grad_std': 0.1250423640012741, 'kl/beta': 0.0006142217316664755, 'kl/avg_steps': 0.53125, 'epoch': 0.98}
|
|||
|
|
98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 666/681 [45:25<00:38, 2.60s/it]
98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 667/681 [45:28<00:36, 2.58s/it]
{'loss': 1.0431, 'grad_norm': 38.44181823730469, 'learning_rate': 7.407554321417764e-10, 'rewards/chosen': -0.320189893245697, 'rewards/rejected': -0.8006361722946167, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.4804462790489197, 'logps/chosen': -567.8262939453125, 'logps/rejected': -1414.545654296875, 'logps/ref_chosen': -43.69598388671875, 'logps/ref_rejected': -93.95926666259766, 'logits/chosen': -3.463103771209717, 'logits/rejected': -4.9513959884643555, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.000606836169026792, 'epsilon_dpo/loss_margin_mean': 796.4560546875, 'epsilon_dpo/beta_margin_mean': 0.4804462492465973, 'epsilon_dpo/beta_margin_std': 0.5962188839912415, 'epsilon_dpo/beta_margin_grad_mean': -0.38951799273490906, 'epsilon_dpo/beta_margin_grad_std': 0.12825334072113037, 'kl/beta': 0.0006109759560786188, 'kl/avg_steps': 0.6875, 'epoch': 0.98}
|
|||
|
|
98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 667/681 [45:28<00:36, 2.58s/it]
98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 668/681 [45:31<00:34, 2.65s/it]
{'loss': 0.9999, 'grad_norm': 37.77009963989258, 'learning_rate': 6.453213851142225e-10, 'rewards/chosen': -0.31520360708236694, 'rewards/rejected': -0.8256018161773682, 'rewards/accuracies': 0.84375, 'rewards/margins': 0.5103981494903564, 'logps/chosen': -569.862548828125, 'logps/rejected': -1479.616943359375, 'logps/ref_chosen': -48.83540344238281, 'logps/ref_rejected': -108.24037170410156, 'logits/chosen': -3.5545034408569336, 'logits/rejected': -4.857883453369141, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0006032615783624351, 'epsilon_dpo/loss_margin_mean': 850.3494262695312, 'epsilon_dpo/beta_margin_mean': 0.5103982090950012, 'epsilon_dpo/beta_margin_std': 0.517078161239624, 'epsilon_dpo/beta_margin_grad_mean': -0.3832368850708008, 'epsilon_dpo/beta_margin_grad_std': 0.11170457303524017, 'kl/beta': 0.0006068041548132896, 'kl/avg_steps': 0.59375, 'epoch': 0.98}
|
|||
|
|
98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 668/681 [45:31<00:34, 2.65s/it]
98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 669/681 [45:33<00:32, 2.67s/it]
{'loss': 1.0343, 'grad_norm': 37.88663101196289, 'learning_rate': 5.564580657695939e-10, 'rewards/chosen': -0.2562495172023773, 'rewards/rejected': -0.7303292751312256, 'rewards/accuracies': 0.828125, 'rewards/margins': 0.4740796983242035, 'logps/chosen': -459.27081298828125, 'logps/rejected': -1305.064208984375, 'logps/ref_chosen': -33.70474624633789, 'logps/ref_rejected': -84.20474243164062, 'logits/chosen': -3.4180922508239746, 'logits/rejected': -4.825163841247559, 'kl/p_epsilon_steps': 0.8125, 'kl/n_epsilon_steps': 0.1875, 'epsilon_dpo/beta': 0.0005995123065076768, 'epsilon_dpo/loss_margin_mean': 795.2932739257812, 'epsilon_dpo/beta_margin_mean': 0.4740796983242035, 'epsilon_dpo/beta_margin_std': 0.5450995564460754, 'epsilon_dpo/beta_margin_grad_mean': -0.3920701742172241, 'epsilon_dpo/beta_margin_grad_std': 0.11787072569131851, 'kl/beta': 0.0006032225210219622, 'kl/avg_steps': 0.625, 'epoch': 0.98}
|
|||
|
|
98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 669/681 [45:33<00:32, 2.67s/it]
98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 670/681 [45:36<00:29, 2.67s/it]
{'loss': 1.0026, 'grad_norm': 38.17559051513672, 'learning_rate': 4.741678157389739e-10, 'rewards/chosen': -0.27749842405319214, 'rewards/rejected': -0.8247913122177124, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.547292947769165, 'logps/chosen': -505.29241943359375, 'logps/rejected': -1491.165283203125, 'logps/ref_chosen': -41.06857681274414, 'logps/ref_rejected': -103.42701721191406, 'logits/chosen': -3.493618965148926, 'logits/rejected': -5.067645072937012, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0005952265928499401, 'epsilon_dpo/loss_margin_mean': 923.514404296875, 'epsilon_dpo/beta_margin_mean': 0.5472928881645203, 'epsilon_dpo/beta_margin_std': 0.6459078788757324, 'epsilon_dpo/beta_margin_grad_mean': -0.3791544735431671, 'epsilon_dpo/beta_margin_grad_std': 0.1323607712984085, 'kl/beta': 0.0005994758103042841, 'kl/avg_steps': 0.71875, 'epoch': 0.98}
|
|||
|
|
98%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 670/681 [45:36<00:29, 2.67s/it]
99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 671/681 [45:38<00:25, 2.60s/it]
{'loss': 1.0497, 'grad_norm': 41.20299530029297, 'learning_rate': 3.9845280344705245e-10, 'rewards/chosen': -0.3173062801361084, 'rewards/rejected': -0.8268150091171265, 'rewards/accuracies': 0.796875, 'rewards/margins': 0.5095087289810181, 'logps/chosen': -567.7605590820312, 'logps/rejected': -1490.3914794921875, 'logps/ref_chosen': -35.17292785644531, 'logps/ref_rejected': -90.90328216552734, 'logits/chosen': -3.6080431938171387, 'logits/rejected': -5.071390151977539, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.000591909047216177, 'epsilon_dpo/loss_margin_mean': 866.9005737304688, 'epsilon_dpo/beta_margin_mean': 0.5095086693763733, 'epsilon_dpo/beta_margin_std': 0.7063105702400208, 'epsilon_dpo/beta_margin_grad_mean': -0.38887372612953186, 'epsilon_dpo/beta_margin_grad_std': 0.14615273475646973, 'kl/beta': 0.0005951978382654488, 'kl/avg_steps': 0.5625, 'epoch': 0.99}
|
|||
|
|
99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎ | 671/681 [45:38<00:25, 2.60s/it]
99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 672/681 [45:41<00:23, 2.63s/it]
{'loss': 1.0379, 'grad_norm': 42.78655242919922, 'learning_rate': 3.293150240547549e-10, 'rewards/chosen': -0.32212620973587036, 'rewards/rejected': -0.8134155869483948, 'rewards/accuracies': 0.8125, 'rewards/margins': 0.491289347410202, 'logps/chosen': -580.21044921875, 'logps/rejected': -1485.48779296875, 'logps/ref_chosen': -35.802879333496094, 'logps/ref_rejected': -100.60040283203125, 'logits/chosen': -3.515069007873535, 'logits/rejected': -5.043622970581055, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.000588598137255758, 'epsilon_dpo/loss_margin_mean': 840.4798583984375, 'epsilon_dpo/beta_margin_mean': 0.491289347410202, 'epsilon_dpo/beta_margin_std': 0.6089746356010437, 'epsilon_dpo/beta_margin_grad_mean': -0.3892599046230316, 'epsilon_dpo/beta_margin_grad_std': 0.13194513320922852, 'kl/beta': 0.0005918685346841812, 'kl/avg_steps': 0.5625, 'epoch': 0.99}
|
|||
|
|
99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 672/681 [45:41<00:23, 2.63s/it]
99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 673/681 [45:44<00:20, 2.59s/it]
{'loss': 0.963, 'grad_norm': 38.58977508544922, 'learning_rate': 2.6675629940689504e-10, 'rewards/chosen': -0.2182503640651703, 'rewards/rejected': -0.7731301784515381, 'rewards/accuracies': 0.9375, 'rewards/margins': 0.5548798441886902, 'logps/chosen': -399.2283935546875, 'logps/rejected': -1416.7021484375, 'logps/ref_chosen': -26.926271438598633, 'logps/ref_rejected': -91.83390808105469, 'logits/chosen': -3.419285297393799, 'logits/rejected': -5.03121280670166, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0005843861144967377, 'epsilon_dpo/loss_margin_mean': 952.566162109375, 'epsilon_dpo/beta_margin_mean': 0.5548798441886902, 'epsilon_dpo/beta_margin_std': 0.5018904209136963, 'epsilon_dpo/beta_margin_grad_mean': -0.3730253279209137, 'epsilon_dpo/beta_margin_grad_std': 0.10759243369102478, 'kl/beta': 0.0005885579157620668, 'kl/avg_steps': 0.71875, 'epoch': 0.99}
|
|||
|
|
99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 673/681 [45:44<00:20, 2.59s/it]
99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 674/681 [45:46<00:17, 2.54s/it]
{'loss': 1.0513, 'grad_norm': 32.84586715698242, 'learning_rate': 2.1077827798404725e-10, 'rewards/chosen': -0.25154006481170654, 'rewards/rejected': -0.7310316562652588, 'rewards/accuracies': 0.796875, 'rewards/margins': 0.479491651058197, 'logps/chosen': -462.48150634765625, 'logps/rejected': -1334.8271484375, 'logps/ref_chosen': -32.161155700683594, 'logps/ref_rejected': -73.33118438720703, 'logits/chosen': -3.5797486305236816, 'logits/rejected': -5.058620452880859, 'kl/p_epsilon_steps': 0.796875, 'kl/n_epsilon_steps': 0.203125, 'epsilon_dpo/beta': 0.0005809462745673954, 'epsilon_dpo/loss_margin_mean': 831.1757202148438, 'epsilon_dpo/beta_margin_mean': 0.47949162125587463, 'epsilon_dpo/beta_margin_std': 0.6231409311294556, 'epsilon_dpo/beta_margin_grad_mean': -0.392082154750824, 'epsilon_dpo/beta_margin_grad_std': 0.13613100349903107, 'kl/beta': 0.0005843578255735338, 'kl/avg_steps': 0.59375, 'epoch': 0.99}
|
|||
|
|
99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 674/681 [45:46<00:17, 2.54s/it]
99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 675/681 [45:48<00:14, 2.48s/it]
{'loss': 0.9896, 'grad_norm': 30.256996154785156, 'learning_rate': 1.6138243485910863e-10, 'rewards/chosen': -0.20505669713020325, 'rewards/rejected': -0.7371607422828674, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.5321040749549866, 'logps/chosen': -380.3410339355469, 'logps/rejected': -1360.566162109375, 'logps/ref_chosen': -26.25046157836914, 'logps/ref_rejected': -80.57308959960938, 'logits/chosen': -3.4077110290527344, 'logits/rejected': -5.058513641357422, 'kl/p_epsilon_steps': 0.890625, 'kl/n_epsilon_steps': 0.109375, 'epsilon_dpo/beta': 0.0005764279630966485, 'epsilon_dpo/loss_margin_mean': 925.9025268554688, 'epsilon_dpo/beta_margin_mean': 0.5321040749549866, 'epsilon_dpo/beta_margin_std': 0.5455756187438965, 'epsilon_dpo/beta_margin_grad_mean': -0.37939831614494324, 'epsilon_dpo/beta_margin_grad_std': 0.11590909957885742, 'kl/beta': 0.0005809086724184453, 'kl/avg_steps': 0.78125, 'epoch': 0.99}
|
|||
|
|
99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 675/681 [45:48<00:14, 2.48s/it]
99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 676/681 [45:51<00:12, 2.49s/it]
{'loss': 1.0612, 'grad_norm': 36.964908599853516, 'learning_rate': 1.1857007165852472e-10, 'rewards/chosen': -0.332570344209671, 'rewards/rejected': -0.7871416211128235, 'rewards/accuracies': 0.828125, 'rewards/margins': 0.4545712471008301, 'logps/chosen': -621.6793823242188, 'logps/rejected': -1472.0751953125, 'logps/ref_chosen': -44.59165954589844, 'logps/ref_rejected': -96.4842300415039, 'logits/chosen': -3.637737512588501, 'logits/rejected': -4.942368507385254, 'kl/p_epsilon_steps': 0.78125, 'kl/n_epsilon_steps': 0.21875, 'epsilon_dpo/beta': 0.0005732206045649946, 'epsilon_dpo/loss_margin_mean': 798.5032958984375, 'epsilon_dpo/beta_margin_mean': 0.45457127690315247, 'epsilon_dpo/beta_margin_std': 0.583877682685852, 'epsilon_dpo/beta_margin_grad_mean': -0.39650610089302063, 'epsilon_dpo/beta_margin_grad_std': 0.13006651401519775, 'kl/beta': 0.0005764055531471968, 'kl/avg_steps': 0.5625, 'epoch': 0.99}
|
|||
|
|
99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 676/681 [45:51<00:12, 2.49s/it]
99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 677/681 [45:53<00:10, 2.52s/it]
{'loss': 0.9904, 'grad_norm': 35.77093505859375, 'learning_rate': 8.23423165278725e-11, 'rewards/chosen': -0.23343393206596375, 'rewards/rejected': -0.7628414630889893, 'rewards/accuracies': 0.921875, 'rewards/margins': 0.5294075012207031, 'logps/chosen': -438.6134948730469, 'logps/rejected': -1427.509521484375, 'logps/ref_chosen': -29.166034698486328, 'logps/ref_rejected': -84.22445678710938, 'logits/chosen': -3.3925907611846924, 'logits/rejected': -5.077644348144531, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0005685811047442257, 'epsilon_dpo/loss_margin_mean': 933.8375244140625, 'epsilon_dpo/beta_margin_mean': 0.5294075012207031, 'epsilon_dpo/beta_margin_std': 0.5406895279884338, 'epsilon_dpo/beta_margin_grad_mean': -0.3787325918674469, 'epsilon_dpo/beta_margin_grad_std': 0.11398439854383469, 'kl/beta': 0.0005731813726015389, 'kl/avg_steps': 0.8125, 'epoch': 0.99}
|
|||
|
|
99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎| 677/681 [45:53<00:10, 2.52s/it]
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 678/681 [45:56<00:07, 2.54s/it]
{'loss': 1.0318, 'grad_norm': 34.48164749145508, 'learning_rate': 5.270012410216185e-11, 'rewards/chosen': -0.24528416991233826, 'rewards/rejected': -0.7404882907867432, 'rewards/accuracies': 0.859375, 'rewards/margins': 0.4952041506767273, 'logps/chosen': -463.51409912109375, 'logps/rejected': -1399.691162109375, 'logps/ref_chosen': -31.562625885009766, 'logps/ref_rejected': -86.20599365234375, 'logits/chosen': -3.5082788467407227, 'logits/rejected': -5.051633358001709, 'kl/p_epsilon_steps': 0.84375, 'kl/n_epsilon_steps': 0.15625, 'epsilon_dpo/beta': 0.0005647094221785665, 'epsilon_dpo/loss_margin_mean': 881.53369140625, 'epsilon_dpo/beta_margin_mean': 0.4952041506767273, 'epsilon_dpo/beta_margin_std': 0.5945535898208618, 'epsilon_dpo/beta_margin_grad_mean': -0.3867654800415039, 'epsilon_dpo/beta_margin_grad_std': 0.12935031950473785, 'kl/beta': 0.0005685618380084634, 'kl/avg_steps': 0.6875, 'epoch': 1.0}
|
|||
|
|
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍| 678/681 [45:56<00:07, 2.54s/it]
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 679/681 [45:58<00:04, 2.48s/it]
{'loss': 1.0791, 'grad_norm': 37.97880935668945, 'learning_rate': 2.9644275480772416e-11, 'rewards/chosen': -0.2744186222553253, 'rewards/rejected': -0.691596269607544, 'rewards/accuracies': 0.796875, 'rewards/margins': 0.41717761754989624, 'logps/chosen': -521.1033935546875, 'logps/rejected': -1316.021484375, 'logps/ref_chosen': -35.110084533691406, 'logps/ref_rejected': -82.25491333007812, 'logits/chosen': -3.43449068069458, 'logits/rejected': -4.8752970695495605, 'kl/p_epsilon_steps': 0.734375, 'kl/n_epsilon_steps': 0.265625, 'epsilon_dpo/beta': 0.0005620888550765812, 'epsilon_dpo/loss_margin_mean': 747.7733154296875, 'epsilon_dpo/beta_margin_mean': 0.41717761754989624, 'epsilon_dpo/beta_margin_std': 0.5460913777351379, 'epsilon_dpo/beta_margin_grad_mean': -0.4052530825138092, 'epsilon_dpo/beta_margin_grad_std': 0.11750102043151855, 'kl/beta': 0.0005646796198561788, 'kl/avg_steps': 0.46875, 'epoch': 1.0}
|
|||
|
|
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 679/681 [45:58<00:04, 2.48s/it]
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 680/681 [46:01<00:02, 2.54s/it]
{'loss': 0.9288, 'grad_norm': 45.923912048339844, 'learning_rate': 1.31753782067201e-11, 'rewards/chosen': -0.24402347207069397, 'rewards/rejected': -0.8650712966918945, 'rewards/accuracies': 0.890625, 'rewards/margins': 0.621047854423523, 'logps/chosen': -484.4205017089844, 'logps/rejected': -1670.13427734375, 'logps/ref_chosen': -49.117393493652344, 'logps/ref_rejected': -118.43990325927734, 'logits/chosen': -3.399845600128174, 'logits/rejected': -4.949437618255615, 'kl/p_epsilon_steps': 0.859375, 'kl/n_epsilon_steps': 0.140625, 'epsilon_dpo/beta': 0.0005580611759796739, 'epsilon_dpo/loss_margin_mean': 1116.3912353515625, 'epsilon_dpo/beta_margin_mean': 0.621047854423523, 'epsilon_dpo/beta_margin_std': 0.5679141879081726, 'epsilon_dpo/beta_margin_grad_mean': -0.360996276140213, 'epsilon_dpo/beta_margin_grad_std': 0.11674246937036514, 'kl/beta': 0.0005620450829155743, 'kl/avg_steps': 0.71875, 'epoch': 1.0}
|
|||
|
|
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 680/681 [46:01<00:02, 2.54s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 681/681 [46:04<00:00, 2.54s/it]
{'loss': 0.9757, 'grad_norm': 39.61741638183594, 'learning_rate': 3.2938662507808745e-12, 'rewards/chosen': -0.2478674352169037, 'rewards/rejected': -0.7954316735267639, 'rewards/accuracies': 0.90625, 'rewards/margins': 0.5475642085075378, 'logps/chosen': -487.08197021484375, 'logps/rejected': -1532.971923828125, 'logps/ref_chosen': -40.42382049560547, 'logps/ref_rejected': -94.08821105957031, 'logits/chosen': -3.4865760803222656, 'logits/rejected': -5.149275302886963, 'kl/p_epsilon_steps': 0.90625, 'kl/n_epsilon_steps': 0.09375, 'epsilon_dpo/beta': 0.0005535554955713451, 'epsilon_dpo/loss_margin_mean': 992.2255859375, 'epsilon_dpo/beta_margin_mean': 0.5475642085075378, 'epsilon_dpo/beta_margin_std': 0.535968005657196, 'epsilon_dpo/beta_margin_grad_mean': -0.37510794401168823, 'epsilon_dpo/beta_margin_grad_std': 0.11336696147918701, 'kl/beta': 0.00055803416762501, 'kl/avg_steps': 0.8125, 'epoch': 1.0}
|
|||
|
|
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 681/681 [46:04<00:00, 2.54s/it][INFO|trainer.py:3984] 2026-04-18 10:17:43,490 >> Saving model checkpoint to /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-681
|
|||
|
|
[INFO|configuration_utils.py:419] 2026-04-18 10:17:43,600 >> Configuration saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-681/config.json
|
|||
|
|
[INFO|configuration_utils.py:911] 2026-04-18 10:17:43,698 >> Configuration saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-681/generation_config.json
|
|||
|
|
[INFO|modeling_utils.py:3580] 2026-04-18 10:18:38,983 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 6 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-681/model.safetensors.index.json.
|
|||
|
|
[INFO|tokenization_utils_base.py:2510] 2026-04-18 10:18:39,006 >> tokenizer config file saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-681/tokenizer_config.json
|
|||
|
|
[INFO|tokenization_utils_base.py:2519] 2026-04-18 10:18:39,011 >> Special tokens file saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-681/special_tokens_map.json
|
|||
|
|
[INFO|trainer.py:4083] 2026-04-18 10:22:03,798 >> Deleting older checkpoint [/scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/checkpoint-400] due to args.save_total_limit
|
|||
|
|
[INFO|trainer.py:2681] 2026-04-18 10:22:07,478 >>
|
|||
|
|
|
|||
|
|
Training completed. Do not forget to share your model on huggingface.co/models =)
|
|||
|
|
|
|||
|
|
|
|||
|
|
{'train_runtime': 3053.5655, 'train_samples_per_second': 14.278, 'train_steps_per_second': 0.223, 'train_loss': 0.6833117403385398, 'epoch': 1.0}
|
|||
|
|
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 681/681 [50:45<00:00, 2.54s/it]
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 681/681 [50:45<00:00, 4.47s/it]
|
|||
|
|
***** train metrics *****
|
|||
|
|
epoch = 1.0
|
|||
|
|
total_flos = 0GF
|
|||
|
|
train_loss = 0.6833
|
|||
|
|
train_runtime = 0:50:53.56
|
|||
|
|
train_samples = 43598
|
|||
|
|
train_samples_per_second = 14.278
|
|||
|
|
train_steps_per_second = 0.223
|
|||
|
|
2026-04-18 10:22:07 - INFO - __main__ - *** Training complete ***
|
|||
|
|
2026-04-18 10:22:07 - INFO - __main__ - *** Save model ***
|
|||
|
|
[INFO|configuration_utils.py:419] 2026-04-18 10:22:23,810 >> Configuration saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/config.json
|
|||
|
|
[INFO|configuration_utils.py:911] 2026-04-18 10:22:23,866 >> Configuration saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/generation_config.json
|
|||
|
|
[INFO|modeling_utils.py:3580] 2026-04-18 10:23:14,522 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 6 checkpoint shards. You can find where each parameters has been saved in the index located at /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/model.safetensors.index.json.
|
|||
|
|
[INFO|tokenization_utils_base.py:2510] 2026-04-18 10:23:14,538 >> tokenizer config file saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/tokenizer_config.json
|
|||
|
|
[INFO|tokenization_utils_base.py:2519] 2026-04-18 10:23:14,542 >> Special tokens file saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/special_tokens_map.json
|
|||
|
|
2026-04-18 10:23:14 - INFO - __main__ - Saved HF-compatible model artifacts to /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332
|
|||
|
|
[INFO|modelcard.py:450] 2026-04-18 10:23:14,816 >> Dropping the following result as it does not have all the necessary fields:
|
|||
|
|
{'dataset': {'name': 'Anthropic/hh-rlhf', 'type': 'Anthropic/hh-rlhf'}}
|
|||
|
|
[INFO|configuration_utils.py:419] 2026-04-18 10:23:14,830 >> Configuration saved in /scratch/feng.yulu/dynamic-dpo-v4/outputs/mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332/config.json
|
|||
|
|
2026-04-18 10:23:14 - INFO - __main__ - *** Evaluate ***
|
|||
|
|
[INFO|trainer.py:4307] 2026-04-18 10:23:14,831 >>
|
|||
|
|
***** Running Evaluation *****
|
|||
|
|
[INFO|trainer.py:4309] 2026-04-18 10:23:14,831 >> Num examples = 2339
|
|||
|
|
[INFO|trainer.py:4312] 2026-04-18 10:23:14,832 >> Batch size = 8
|
|||
|
|
0%| | 0/73 [00:00<?, ?it/s]
3%|███▏ | 2/73 [00:00<00:18, 3.87it/s]
4%|████▊ | 3/73 [00:01<00:27, 2.57it/s]
5%|██████▍ | 4/73 [00:01<00:30, 2.25it/s]
7%|████████ | 5/73 [00:02<00:32, 2.10it/s]
8%|█████████▌ | 6/73 [00:02<00:32, 2.03it/s]
10%|███████████▏ | 7/73 [00:03<00:32, 2.02it/s]
11%|████████████▊ | 8/73 [00:03<00:31, 2.07it/s]
12%|██████████████▍ | 9/73 [00:04<00:31, 2.01it/s]
14%|███████████████▉ | 10/73 [00:04<00:32, 1.96it/s]
15%|█████████████████▍ | 11/73 [00:05<00:31, 1.97it/s]
16%|███████████████████ | 12/73 [00:05<00:30, 1.97it/s]
18%|████████████████████▋ | 13/73 [00:06<00:31, 1.91it/s]
19%|██████████████████████▏ | 14/73 [00:06<00:30, 1.91it/s]
21%|███████████████████████▊ | 15/73 [00:07<00:31, 1.87it/s]
22%|█████████████████████████▍ | 16/73 [00:07<00:30, 1.85it/s]
23%|███████████████████████████ | 17/73 [00:08<00:29, 1.87it/s]
25%|████████████████████████████▌ | 18/73 [00:08<00:29, 1.85it/s]
26%|██████████████████████████████▏ | 19/73 [00:09<00:29, 1.84it/s]
27%|███████████████████████████████▊ | 20/73 [00:10<00:29, 1.82it/s]
29%|█████████████████████████████████▎ | 21/73 [00:10<00:28, 1.82it/s]
30%|██████████████████████████████████▉
|
|||
|
|
***** eval metrics *****
|
|||
|
|
epoch = 1.0
|
|||
|
|
eval_epsilon_dpo/beta = 0.0006
|
|||
|
|
eval_epsilon_dpo/beta_margin_grad_mean = -0.4373
|
|||
|
|
eval_epsilon_dpo/beta_margin_grad_std = 0.1302
|
|||
|
|
eval_epsilon_dpo/beta_margin_mean = 0.2754
|
|||
|
|
eval_epsilon_dpo/beta_margin_std = 0.5839
|
|||
|
|
eval_epsilon_dpo/loss_margin_mean = 506.5054
|
|||
|
|
eval_kl/n_epsilon_steps = 0.3275
|
|||
|
|
eval_kl/p_epsilon_steps = 0.6725
|
|||
|
|
eval_logits/chosen = -4.0524
|
|||
|
|
eval_logits/rejected = -4.9681
|
|||
|
|
eval_logps/chosen = -881.2797
|
|||
|
|
eval_logps/ref_chosen = -68.2911
|
|||
|
|
eval_logps/ref_rejected = -92.0804
|
|||
|
|
eval_logps/rejected = -1411.5745
|
|||
|
|
eval_loss = 0.6067
|
|||
|
|
eval_rewards/accuracies = 0.6922
|
|||
|
|
eval_rewards/chosen = -0.4507
|
|||
|
|
eval_rewards/margins = 0.2754
|
|||
|
|
eval_rewards/rejected = -0.7261
|
|||
|
|
eval_runtime = 0:00:38.41
|
|||
|
|
eval_samples = 2339
|
|||
|
|
eval_samples_per_second = 60.884
|
|||
|
|
eval_steps_per_second = 1.926
|
|||
|
|
2026-04-18 10:23:53 - INFO - __main__ - *** Training complete! ***
|
|||
|
|
wandb: - 0.014 MB of 0.014 MB uploaded
wandb: \ 0.014 MB of 0.014 MB uploaded
wandb: | 0.014 MB of 0.014 MB uploaded
wandb: / 0.049 MB of 0.351 MB uploaded (0.002 MB deduped)
wandb: - 0.353 MB of 0.353 MB uploaded (0.002 MB deduped)
wandb:
|
|||
|
|
wandb: Run history:
|
|||
|
|
wandb: eval/epsilon_dpo/beta █▄▂▂▁▁▁
|
|||
|
|
wandb: eval/epsilon_dpo/beta_margin_grad_mean ▆█▇▄▂▂▁
|
|||
|
|
wandb: eval/epsilon_dpo/beta_margin_grad_std █▇▅▄▁▄▁
|
|||
|
|
wandb: eval/epsilon_dpo/beta_margin_mean ▇█▅▃▂▂▁
|
|||
|
|
wandb: eval/epsilon_dpo/beta_margin_std █▇▄▃▁▂▁
|
|||
|
|
wandb: eval/epsilon_dpo/loss_margin_mean ▁▂▂▃▄██
|
|||
|
|
wandb: eval/kl/n_epsilon_steps ▅▁▁▄▄▇█
|
|||
|
|
wandb: eval/kl/p_epsilon_steps ▄██▅▅▂▁
|
|||
|
|
wandb: eval/logits/chosen ▄▇█▇▅▂▁
|
|||
|
|
wandb: eval/logits/rejected ▅▇█▇▅▂▁
|
|||
|
|
wandb: eval/logps/chosen █▇▇▆▅▂▁
|
|||
|
|
wandb: eval/logps/ref_chosen ▁▁▁▁▁▁▁
|
|||
|
|
wandb: eval/logps/ref_rejected ▁▁▁▁▁▁▁
|
|||
|
|
wandb: eval/logps/rejected █▇▇▆▅▂▁
|
|||
|
|
wandb: eval/loss ▄▁▁▄▆▇█
|
|||
|
|
wandb: eval/rewards/accuracies ▅██▅▅▂▁
|
|||
|
|
wandb: eval/rewards/chosen ▄▁▄▆▇▇█
|
|||
|
|
wandb: eval/rewards/margins ▇█▅▃▂▂▁
|
|||
|
|
wandb: eval/rewards/rejected ▃▁▄▆▇▇█
|
|||
|
|
wandb: eval/runtime █▁▃▁▄▃▅
|
|||
|
|
wandb: eval/samples_per_second ▁█▆█▅▆▄
|
|||
|
|
wandb: eval/steps_per_second ▁█▇█▅▇▄
|
|||
|
|
wandb: train/epoch ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███
|
|||
|
|
wandb: train/epsilon_dpo/beta █▇▇▆▅▅▄▄▃▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
|
|||
|
|
wandb: train/epsilon_dpo/beta_margin_grad_mean ▁▁▄▆▇▇▇▇▇▇█▇█▇█▇▇▆▆▇▆▆▅▆▆▅▅▅▄▄▄▄▅▄▄▄▄▄▄▄
|
|||
|
|
wandb: train/epsilon_dpo/beta_margin_grad_std ▁▁▄█▆▆▆▇▆▇▆▆▅▆▅▅▆▆▆▆▅▄▅▅▅▅▅▅▄▄▄▄▆▆▅▅▆▅▅▄
|
|||
|
|
wandb: train/epsilon_dpo/beta_margin_mean ▁▁▃▆▇▇█▇███▇█▆▇▆▆▅▅▅▅▄▄▄▄▃▃▃▃▃▃▃▄▃▃▃▃▃▃▂
|
|||
|
|
wandb: train/epsilon_dpo/beta_margin_std ▁▁▃▇▇▆▇█▇█▇▇▆▆▅▅▅▅▅▄▄▃▃▃▃▃▃▃▃▃▂▃▄▄▃▃▃▃▃▃
|
|||
|
|
wandb: train/epsilon_dpo/loss_margin_mean ▁▁▁▁▁▁▁▁▁▂▂▂▂▂▂▂▂▂▂▂▂▂▂▃▃▃▃▃▃▄▃▄▆▆▅▆▆▇█▇
|
|||
|
|
wandb: train/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███
|
|||
|
|
wandb: train/grad_norm ██▄▆▃▂▃▃▃▄▂▃▂▂▁▃▂▂▂▂▂▂▂▂▂▂▁▁▁▂▁▂▁▂▂▂▂▂▂▂
|
|||
|
|
wandb: train/kl/avg_steps ▁▇▇▆▇▇▇▆▇▆▇▇▇▇█▇▇▇▇▇▇█▇▇▇▇▇▇▇▇▇▇▇▆▆▇▆▇▇▇
|
|||
|
|
wandb: train/kl/beta ██▇▆▆▅▅▄▄▃▃▃▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
|
|||
|
|
wandb: train/kl/n_epsilon_steps █▂▂▃▂▂▂▃▂▃▂▂▂▂▁▂▂▂▂▂▂▁▂▂▂▂▂▂▂▂▂▂▂▃▃▂▃▂▂▂
|
|||
|
|
wandb: train/kl/p_epsilon_steps ▁▇▇▆▇▇▇▇▇▇▇▇▇▇█▇▇▇▇▇▇█▇▇▇▇▇▇▇▇▇▇▇▆▆▇▆▇▇▇
|
|||
|
|
wandb: train/learning_rate ▂▃▅▇██████▇▇▇▇▇▆▆▆▆▅▅▅▄▄▄▄▃▃▃▂▂▂▂▂▁▁▁▁▁▁
|
|||
|
|
wandb: train/logits/chosen ▁▂▁▂▂▂▃▄▅▅▆▆▆▆▇██▇▇▆▆▆▆▅▅▅▅▄▄▄▃▂▂▁▁▂▂▂▂▂
|
|||
|
|
wandb: train/logits/rejected ▄▄▄▄▅▅▅▆▆▆▇▇▇▇▇████▇▇▇▇▇▇▆▆▅▅▅▄▃▂▂▂▂▁▁▁▁
|
|||
|
|
wandb: train/logps/chosen ███████▇▇▇▇▇▇▇▇▇▆▆▆▆▆▆▅▅▆▆▅▅▆▅▅▅▁▁▂▄▁▂▂▁
|
|||
|
|
wandb: train/logps/ref_chosen ▅█▆▇▅▆▆▆▅▄▆▄▆▅█▅▅▅▇▆▅▇▆▃▅▅▅▁▆▄▄▇▅▆▇█▄▅▅▃
|
|||
|
|
wandb: train/logps/ref_rejected ▇▄█▅▅▂▅█▇▄▄▃▅▇▅▅▄▅█▅█▄▇▃▅▄▃▁▇▃▃▆▃▄█▅▄▂▂▂
|
|||
|
|
wandb: train/logps/rejected ████████▇▇▇▇▇▇▇▇▇▇▇▆▇▆▆▆▆▆▆▅▆▅▅▅▂▃▃▃▂▂▁▁
|
|||
|
|
wandb: train/loss █▇▅▄▂▂▂▂▂▂▁▂▁▂▁▂▂▃▃▂▃▂▃▃▃▄▄▄▅▄▅▅▄▅▅▅▅▅▄▅
|
|||
|
|
wandb: train/rewards/accuracies ▁▇█▇█▇▇▇▇▇▇▇████▇▇▇▇▇█▇▇▇▇▇▇▇▇▇▇▇▆▇▇▇▇▇▇
|
|||
|
|
wandb: train/rewards/chosen ███▆▅▆▆▄▃▃▁▂▃▄▄▅▃▄▅▅▅▆▅▆▆▆▇▆▇▇▇▇▆▆▇▇▇▇▇▇
|
|||
|
|
wandb: train/rewards/margins ▁▁▃▆▇▇█▇███▇█▆▇▆▆▅▅▅▅▄▄▄▄▃▃▃▃▃▃▃▄▃▃▃▃▃▃▂
|
|||
|
|
wandb: train/rewards/rejected ██▇▅▃▄▃▃▂▂▁▂▂▃▃▃▃▄▄▄▅▅▅▅▆▆▆▆▇▆▇▇▆▆▆▇▆▇▇▇
|
|||
|
|
wandb:
|
|||
|
|
wandb: Run summary:
|
|||
|
|
wandb: eval/epsilon_dpo/beta 0.00055
|
|||
|
|
wandb: eval/epsilon_dpo/beta_margin_grad_mean -0.43731
|
|||
|
|
wandb: eval/epsilon_dpo/beta_margin_grad_std 0.13021
|
|||
|
|
wandb: eval/epsilon_dpo/beta_margin_mean 0.27542
|
|||
|
|
wandb: eval/epsilon_dpo/beta_margin_std 0.58391
|
|||
|
|
wandb: eval/epsilon_dpo/loss_margin_mean 506.5054
|
|||
|
|
wandb: eval/kl/n_epsilon_steps 0.32748
|
|||
|
|
wandb: eval/kl/p_epsilon_steps 0.67252
|
|||
|
|
wandb: eval/logits/chosen -4.05236
|
|||
|
|
wandb: eval/logits/rejected -4.96813
|
|||
|
|
wandb: eval/logps/chosen -881.27972
|
|||
|
|
wandb: eval/logps/ref_chosen -68.29111
|
|||
|
|
wandb: eval/logps/ref_rejected -92.08038
|
|||
|
|
wandb: eval/logps/rejected -1411.57446
|
|||
|
|
wandb: eval/loss 0.60668
|
|||
|
|
wandb: eval/rewards/accuracies 0.69221
|
|||
|
|
wandb: eval/rewards/chosen -0.45067
|
|||
|
|
wandb: eval/rewards/margins 0.27542
|
|||
|
|
wandb: eval/rewards/rejected -0.7261
|
|||
|
|
wandb: eval/runtime 38.4171
|
|||
|
|
wandb: eval/samples_per_second 60.884
|
|||
|
|
wandb: eval/steps_per_second 1.926
|
|||
|
|
wandb: total_flos 0.0
|
|||
|
|
wandb: train/epoch 1.0
|
|||
|
|
wandb: train/epsilon_dpo/beta 0.00055
|
|||
|
|
wandb: train/epsilon_dpo/beta_margin_grad_mean -0.37511
|
|||
|
|
wandb: train/epsilon_dpo/beta_margin_grad_std 0.11337
|
|||
|
|
wandb: train/epsilon_dpo/beta_margin_mean 0.54756
|
|||
|
|
wandb: train/epsilon_dpo/beta_margin_std 0.53597
|
|||
|
|
wandb: train/epsilon_dpo/loss_margin_mean 992.22559
|
|||
|
|
wandb: train/global_step 681
|
|||
|
|
wandb: train/grad_norm 39.61742
|
|||
|
|
wandb: train/kl/avg_steps 0.8125
|
|||
|
|
wandb: train/kl/beta 0.00056
|
|||
|
|
wandb: train/kl/n_epsilon_steps 0.09375
|
|||
|
|
wandb: train/kl/p_epsilon_steps 0.90625
|
|||
|
|
wandb: train/learning_rate 0.0
|
|||
|
|
wandb: train/logits/chosen -3.48658
|
|||
|
|
wandb: train/logits/rejected -5.14928
|
|||
|
|
wandb: train/logps/chosen -487.08197
|
|||
|
|
wandb: train/logps/ref_chosen -40.42382
|
|||
|
|
wandb: train/logps/ref_rejected -94.08821
|
|||
|
|
wandb: train/logps/rejected -1532.97192
|
|||
|
|
wandb: train/loss 0.9757
|
|||
|
|
wandb: train/rewards/accuracies 0.90625
|
|||
|
|
wandb: train/rewards/chosen -0.24787
|
|||
|
|
wandb: train/rewards/margins 0.54756
|
|||
|
|
wandb: train/rewards/rejected -0.79543
|
|||
|
|
wandb: train_loss 0.68331
|
|||
|
|
wandb: train_runtime 3053.5655
|
|||
|
|
wandb: train_samples_per_second 14.278
|
|||
|
|
wandb: train_steps_per_second 0.223
|
|||
|
|
wandb:
|
|||
|
|
wandb: 🚀 View run mistral-7b-base-epsilon-dpo-hh-helpful-4xh200-batch-64-20260418-015332 at: https://wandb.ai/can-not-fand-northeastern-university/ood-run-4xh200/runs/7dwx2wac
|
|||
|
|
wandb: ⭐️ View project at: https://wandb.ai/can-not-fand-northeastern-university/ood-run-4xh200
|
|||
|
|
wandb: Synced 6 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)
|
|||
|
|
wandb: Find logs at: /scratch/feng.yulu/dynamic-dpo-v4/wandb/wandb/run-20260418_093116-7dwx2wac/logs
|
|||
|
|
wandb: WARNING The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require("core")`! See https://wandb.me/wandb-core for more information.
|