[2026-04-16 10:31:52,083] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:135] baseline 0.000GB ()
[2026-04-16 10:31:52,086] [INFO] [axolotl.cli.config.load_cfg:341] [PID:135] config:
{
  "activation_offloading": false,
  "axolotl_config_path": "./config.yaml",
  "base_model": "Qwen/Qwen2.5-7B",
  "base_model_config": "Qwen/Qwen2.5-7B",
  "batch_size": 16,
  "bf16": true,
  "capabilities": {
    "bf16": true,
    "compute_capability": "sm_80",
    "fp8": false,
    "n_gpu": 1,
    "n_node": 1,
    "tf32": true
  },
  "context_parallel_size": 1,
  "dataloader_num_workers": 1,
  "dataloader_pin_memory": true,
  "dataloader_prefetch_factor": 256,
  "dataset_num_proc": 96,
  "dataset_prepared_path": "./prepared_data",
  "datasets": [
    {
      "path": "Bossssss/medqa-dpo-idk",
      "split": "train",
      "type": "passthrough"
    }
  ],
  "ddp": false,
  "device": "cuda:0",
  "dion_rank_fraction": 1.0,
  "dion_rank_multiple_of": 1,
  "eaft_alpha": 1.0,
  "eaft_k": 20,
  "env_capabilities": {
    "torch_version": "2.9.1"
  },
  "eval_batch_size": 2,
  "eval_causal_lm_metrics": [
    "sacrebleu",
    "comet",
    "ter",
    "chrf"
  ],
  "eval_max_new_tokens": 128,
  "eval_steps": 100,
  "eval_strategy": "steps",
  "eval_table_size": 0,
  "experimental_skip_move_to_device": true,
  "flash_attention": true,
  "fp16": false,
  "generate_samples": false,
  "generation_do_sample": true,
  "generation_max_new_tokens": 50,
  "generation_prompt_ratio": 0.5,
  "generation_temperature": 0.7,
  "gradient_accumulation_steps": 8,
  "gradient_checkpointing": true,
  "greater_is_better": false,
  "include_tkps": true,
  "is_falcon_derived_model": false,
  "is_llama_derived_model": false,
  "is_mistral_derived_model": false,
  "layer_offloading": false,
  "learning_rate": 5e-05,
  "lisa_layers_attribute": "model.layers",
  "load_best_model_at_end": true,
  "load_in_4bit": false,
  "load_in_8bit": false,
  "local_rank": 0,
  "logging_steps": 10,
  "lora_dropout": 0.0,
  "loraplus_lr_embedding": 1e-06,
  "lr_scheduler": "cosine",
  "max_grad_norm": 1.0,
  "mean_resizing_embeddings": false,
  "merge_method": "memory_efficient",
  "metric_for_best_model": "eval_loss",
  "micro_batch_size": 2,
  "model_config_type": "qwen2",
  "num_epochs": 3.0,
  "num_generation_samples": 3,
  "optimizer": "adamw_bnb_8bit",
  "otel_metrics_host": "localhost",
  "otel_metrics_port": 8000,
  "output_dir": "kikiyaa/qwen-dpo-finetuned-ver2",
  "pretrain_multipack_attn": true,
  "profiler_steps_start": 0,
  "qlora_sharded_model_loading": false,
  "quantize_moe_experts": false,
  "ray_num_workers": 1,
  "resources_per_worker": {
    "GPU": 1
  },
  "rl": "dpo",
  "rl_beta": 0.1,
  "sample_packing": false,
  "sample_packing_bin_size": 200,
  "sample_packing_group_size": 100000,
  "save_only_model": false,
  "save_safetensors": true,
  "save_steps": 100,
  "save_total_limit": 3,
  "seed": 42,
  "sequence_len": 2048,
  "shuffle_before_merging_datasets": false,
  "shuffle_merged_datasets": true,
  "skip_prepare_dataset": false,
  "streaming_multipack_buffer_size": 10000,
  "strict": false,
  "tensor_parallel_size": 1,
  "tf32": true,
  "tiled_mlp_use_original_mlp": true,
  "tokenizer_config": "Qwen/Qwen2.5-7B",
  "tokenizer_save_jinja_files": true,
  "tokenizer_type": "AutoTokenizer",
  "torch_dtype": "torch.bfloat16",
  "train_on_inputs": false,
  "trl": {
    "async_prefetch": false,
    "log_completions": false,
    "mask_truncated_completions": false,
    "ref_model_mixup_alpha": 0.9,
    "ref_model_sync_steps": 64,
    "replay_buffer_size": 0,
    "replay_recompute_logps": true,
    "reroll_max_groups": 1,
    "reroll_start_fraction": 1.0,
    "reward_num_workers": 1,
    "scale_rewards": true,
    "skip_zero_advantage_batches": true,
    "sync_ref_model": false,
    "use_data_producer": false,
    "use_vllm": false,
    "vllm_lora_sync": false,
    "vllm_server_host": "0.0.0.0",
    "vllm_server_port": 8000
  },
  "type_of_model": "AutoModelForCausalLM",
  "use_otel_metrics": false,
  "use_ray": false,
  "val_set_size": 0.05,
  "vllm": {
    "device": "auto",
    "dtype": "auto",
    "gpu_memory_utilization": 0.9,
    "host": "0.0.0.0",
    "port": 8000
  },
  "warmup_ratio": 0.1,
  "weight_decay": 0.01,
  "world_size": 1
}
[2026-04-16 10:31:57,826] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:135] EOS: 151643 / <|endoftext|>
[2026-04-16 10:31:57,826] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:135] BOS: None / None
[2026-04-16 10:31:57,826] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:135] PAD: 151643 / <|endoftext|>
[2026-04-16 10:31:57,826] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:135] UNK: None / None
[2026-04-16 10:31:57,831] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:475] [PID:135] Loading prepared dataset from disk at prepared_data/b845853b1ffa0f92dd415602f19f08af...
[2026-04-16 10:31:57,902] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:135] loading tokenizer... Qwen/Qwen2.5-7B
[2026-04-16 10:31:59,556] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:135] EOS: 151643 / <|endoftext|>
[2026-04-16 10:31:59,556] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:135] BOS: None / None
[2026-04-16 10:31:59,556] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:135] PAD: 151643 / <|endoftext|>
[2026-04-16 10:31:59,556] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:135] UNK: None / None
[2026-04-16 10:31:59,556] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:135] Loading model
[2026-04-16 10:31:59,777] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:135] Patched OptimState8bit for torch.compile compatibility
[2026-04-16 10:31:59,778] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:135] Patched OptimState4bit for torch.compile compatibility
[2026-04-16 10:31:59,778] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:135] Patched OptimStateFp8 for torch.compile compatibility
[2026-04-16 10:31:59,795] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:135] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-04-16 10:31:59,798] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:135] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
[2026-04-16 10:32:00,017] [INFO] [axolotl.loaders.patch_manager._apply_trl_trainer_utils_patches:852] [PID:135] Patched trl.trainer.utils with Triton selective_log_softmax and entropy_from_logits

Fetching 4 files:   0%|          | 0/4 [00:00<?, ?it/s]
Fetching 4 files:  25%|██▌       | 1/4 [02:20<07:01, 140.64s/it]
Fetching 4 files:  75%|███████▌  | 3/4 [02:22<00:37, 37.05s/it] 
Fetching 4 files: 100%|██████████| 4/4 [02:22<00:00, 35.56s/it]

Loading weights:   0%|          | 0/339 [00:00<?, ?it/s]
Loading weights: 100%|██████████| 339/339 [00:00<00:00, 5017.02it/s]
[2026-04-16 10:34:26,588] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:361] [PID:135] Converting modules to torch.bfloat16
[2026-04-16 10:34:27,756] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:135] Memory usage after model load 0.000GB ()
[2026-04-16 10:34:28,584] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:75] [PID:135] Patched OptimState8bit for torch.compile compatibility
[2026-04-16 10:34:28,584] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:122] [PID:135] Patched OptimState4bit for torch.compile compatibility
[2026-04-16 10:34:28,584] [DEBUG] [axolotl.monkeypatch.torchao_optim.patch_torchao_optim_state_8bit:154] [PID:135] Patched OptimStateFp8 for torch.compile compatibility
[2026-04-16 10:34:28,584] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:51] [PID:135] Trainer.evaluation_loop already patched
[2026-04-16 10:34:28,584] [INFO] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:110] [PID:135] Trainer._maybe_log_save_evaluate already patched
[2026-04-16 10:34:28,584] [INFO] [axolotl.loaders.patch_manager._apply_trl_trainer_utils_patches:852] [PID:135] Patched trl.trainer.utils with Triton selective_log_softmax and entropy_from_logits

Loading weights:   0%|          | 0/339 [00:00<?, ?it/s]
Loading weights: 100%|██████████| 339/339 [00:00<00:00, 4331.59it/s]
[2026-04-16 10:34:30,928] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:361] [PID:135] Converting modules to torch.bfloat16
[2026-04-16 10:34:32,130] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:135] Memory usage after model load 0.000GB ()

Adding EOS to train dataset (num_proc=96):   0%|          | 0/5399 [00:00<?, ? examples/s]
Adding EOS to train dataset (num_proc=96):   1%|          | 57/5399 [00:06<09:40,  9.20 examples/s]
Adding EOS to train dataset (num_proc=96):   5%|▌         | 285/5399 [00:06<01:24, 60.34 examples/s]
Adding EOS to train dataset (num_proc=96):  18%|█▊        | 969/5399 [00:06<00:16, 271.57 examples/s]
Adding EOS to train dataset (num_proc=96):  76%|███████▌  | 4111/5399 [00:06<00:00, 1596.87 examples/s]
Adding EOS to train dataset (num_proc=96): 100%|██████████| 5399/5399 [00:07<00:00, 700.21 examples/s] 

Tokenizing train dataset (num_proc=96):   0%|          | 0/5399 [00:00<?, ? examples/s][2026-04-16 10:34:51,717] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:672] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:51,764] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:672] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:51,764] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:672] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:51,773] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:672] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:51,773] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:672] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):   1%|          | 27/5399 [00:08<27:02,  3.31 examples/s][2026-04-16 10:34:52,858] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:673] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:52,859] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:673] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:52,899] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:673] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:52,899] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:673] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):   2%|▏         | 85/5399 [00:09<07:52, 11.24 examples/s][2026-04-16 10:34:52,923] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:673] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:53,960] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:674] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:53,960] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:674] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:53,964] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:674] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:53,970] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:674] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:53,970] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:674] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:53,978] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:674] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:54,007] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:674] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:54,007] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:674] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):   3%|▎         | 139/5399 [00:10<04:47, 18.29 examples/s][2026-04-16 10:34:54,014] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:674] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:54,014] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:674] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:54,032] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:674] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:54,032] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:674] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:55,060] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:675] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:55,061] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:675] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:55,065] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:675] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:55,108] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:675] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:55,108] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:675] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):   4%|▎         | 198/5399 [00:11<03:22, 25.71 examples/s][2026-04-16 10:34:55,126] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:675] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:55,126] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:675] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:56,178] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:676] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:56,178] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:676] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:56,206] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:676] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:56,206] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:676] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:56,208] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:676] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:56,208] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:676] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:56,241] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:676] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:56,241] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:676] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):   5%|▍         | 254/5399 [00:12<02:44, 31.37 examples/s][2026-04-16 10:34:56,254] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:676] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:56,254] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:676] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:57,314] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:677] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:57,314] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:677] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:57,316] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:677] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:57,316] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:677] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):   6%|▌         | 311/5399 [00:13<02:19, 36.35 examples/s][2026-04-16 10:34:57,356] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:677] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:57,357] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:677] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:57,368] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:677] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:57,376] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:677] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:57,377] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:677] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:58,407] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:678] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:58,407] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:678] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:58,412] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:678] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:58,412] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:678] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:58,437] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:678] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:58,437] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:678] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):   7%|▋         | 367/5399 [00:14<02:05, 39.95 examples/s][2026-04-16 10:34:58,479] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:678] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:58,500] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:678] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:59,541] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:679] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:59,541] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:679] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:59,568] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:679] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:34:59,568] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:679] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):   8%|▊         | 425/5399 [00:15<01:54, 43.41 examples/s][2026-04-16 10:34:59,584] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:679] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,615] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,615] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,627] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,627] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,631] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,648] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,648] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,652] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,672] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,672] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):   9%|▉         | 481/5399 [00:17<01:48, 45.46 examples/s][2026-04-16 10:35:00,703] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,703] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,714] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:00,714] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:680] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:01,765] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:681] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:01,766] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:681] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:01,775] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:681] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:01,775] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:681] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:01,797] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:681] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:01,797] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:681] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  10%|▉         | 538/5399 [00:18<01:43, 46.82 examples/s][2026-04-16 10:35:01,822] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:681] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:01,822] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:681] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:01,837] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:681] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:01,843] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:681] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:01,843] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:681] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:02,864] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:682] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:02,878] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:682] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:02,894] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:682] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:02,894] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:682] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:02,905] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:682] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:02,905] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:682] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  11%|█         | 598/5399 [00:19<01:38, 48.97 examples/s][2026-04-16 10:35:02,947] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:682] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:02,948] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:682] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:02,951] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:682] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:02,951] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:682] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:03,960] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:683] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:03,962] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:683] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:03,982] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:683] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:03,982] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:683] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:04,000] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:683] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:04,000] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:683] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:04,006] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:683] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:04,006] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:683] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:04,008] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:683] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:04,009] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:683] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  12%|█▏        | 654/5399 [00:20<01:35, 49.61 examples/s][2026-04-16 10:35:04,028] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:683] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:04,028] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:683] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:05,066] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:684] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:05,066] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:684] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:05,092] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:684] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:05,092] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:684] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:05,094] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:684] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:05,094] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:684] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  13%|█▎        | 711/5399 [00:21<01:33, 50.22 examples/s][2026-04-16 10:35:06,168] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:685] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:06,168] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:685] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:06,172] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:685] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:06,175] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:685] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:06,175] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:685] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:06,207] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:685] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:06,207] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:685] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:06,215] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:685] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:06,215] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:685] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  14%|█▍        | 767/5399 [00:22<01:32, 50.28 examples/s][2026-04-16 10:35:06,244] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:685] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:06,244] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:685] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:07,269] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:686] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:07,270] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:686] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:07,271] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:686] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:07,272] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:686] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:07,294] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:686] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:07,294] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:686] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:07,303] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:686] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  15%|█▌        | 825/5399 [00:23<01:29, 51.13 examples/s][2026-04-16 10:35:07,324] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:686] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:07,324] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:686] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,359] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,360] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,374] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,374] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,384] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,384] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,386] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,386] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,392] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,393] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,415] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,415] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,417] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,417] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  16%|█▋        | 880/5399 [00:24<01:29, 50.74 examples/s][2026-04-16 10:35:08,424] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,424] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,437] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,437] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:08,445] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:687] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:09,490] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:688] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:09,490] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:688] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:09,492] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:688] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:09,499] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:688] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:09,499] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:688] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  17%|█▋        | 937/5399 [00:25<01:27, 50.97 examples/s][2026-04-16 10:35:09,536] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:688] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:09,536] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:688] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:09,553] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:688] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:10,590] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:689] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:10,591] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:689] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:10,593] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:689] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:10,594] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:689] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  18%|█▊        | 994/5399 [00:27<01:25, 51.23 examples/s][2026-04-16 10:35:10,644] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:689] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:10,644] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:689] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:10,657] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:689] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:10,657] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:689] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:11,635] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:690] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:11,673] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:690] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:11,673] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:690] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  19%|█▉        | 1052/5399 [00:28<01:23, 52.30 examples/s][2026-04-16 10:35:11,697] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:690] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:11,697] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:690] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:11,710] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:690] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:11,710] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:690] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:12,809] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:691] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:12,810] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:691] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  21%|██        | 1109/5399 [00:29<01:23, 51.61 examples/s][2026-04-16 10:35:12,823] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:691] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:12,823] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:691] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:12,849] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:691] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:12,849] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:691] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:13,876] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:692] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:13,876] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:692] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:13,879] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:692] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:13,879] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:692] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:13,904] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:692] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:13,904] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:692] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:13,906] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:692] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:13,906] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:692] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:13,922] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:692] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  22%|██▏       | 1167/5399 [00:30<01:21, 51.87 examples/s][2026-04-16 10:35:15,019] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:693] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  23%|██▎       | 1221/5399 [00:31<01:21, 51.00 examples/s][2026-04-16 10:35:15,069] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:693] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:16,040] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:694] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:16,040] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:694] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:16,074] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:694] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:16,078] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:694] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:16,078] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:694] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  24%|██▍       | 1283/5399 [00:32<01:17, 53.29 examples/s][2026-04-16 10:35:16,111] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:694] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:16,111] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:694] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:17,180] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:695] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:17,180] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:695] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  25%|██▍       | 1338/5399 [00:33<01:18, 51.83 examples/s][2026-04-16 10:35:17,217] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:695] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:17,217] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:695] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:17,246] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:695] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:17,246] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:695] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:18,262] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:696] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:18,262] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:696] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:18,284] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:696] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:18,284] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:696] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:18,294] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:696] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  26%|██▌       | 1392/5399 [00:34<01:18, 51.01 examples/s][2026-04-16 10:35:18,320] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:696] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:18,333] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:696] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:18,333] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:696] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:19,354] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:697] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:19,369] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:697] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:19,369] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:697] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  27%|██▋       | 1449/5399 [00:35<01:16, 51.38 examples/s][2026-04-16 10:35:19,412] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:697] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:19,418] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:697] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:19,418] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:697] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:19,420] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:697] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:20,436] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:20,436] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:20,440] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:20,440] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:20,457] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:20,461] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:20,467] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:20,467] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:20,488] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:20,488] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:20,493] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:20,493] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  28%|██▊       | 1504/5399 [00:36<01:16, 51.10 examples/s][2026-04-16 10:35:20,515] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:698] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:21,536] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:21,539] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:21,539] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:21,573] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:21,573] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:21,582] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:21,582] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  29%|██▉       | 1563/5399 [00:37<01:13, 51.99 examples/s][2026-04-16 10:35:21,592] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:21,592] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:21,603] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:21,603] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:21,610] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:21,610] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:699] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:22,636] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:700] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:22,658] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:700] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:22,658] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:700] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:22,680] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:700] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:22,680] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:700] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  30%|██▉       | 1617/5399 [00:39<01:14, 51.09 examples/s][2026-04-16 10:35:22,691] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:700] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:22,691] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:700] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:22,693] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:700] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:22,693] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:700] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:22,717] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:700] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:22,717] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:700] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:23,757] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:701] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:23,758] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:701] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:23,766] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:701] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  31%|███       | 1673/5399 [00:40<01:12, 51.34 examples/s][2026-04-16 10:35:23,770] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:701] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:23,770] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:701] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:23,774] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:701] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:23,774] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:701] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:23,785] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:701] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:24,841] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:702] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:24,841] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:702] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:24,843] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:702] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:24,867] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:702] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  32%|███▏      | 1728/5399 [00:41<01:12, 50.65 examples/s][2026-04-16 10:35:24,892] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:702] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:24,892] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:702] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:24,904] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:702] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:24,904] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:702] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:24,913] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:702] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:24,913] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:702] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:25,977] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:25,977] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:25,996] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:25,996] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:26,005] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:26,006] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:26,016] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:26,016] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:26,017] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:26,017] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:26,021] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:26,021] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  33%|███▎      | 1785/5399 [00:42<01:11, 50.29 examples/s][2026-04-16 10:35:26,041] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:26,041] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:26,058] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:26,058] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:703] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,090] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,096] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,096] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,097] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,097] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,105] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,105] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,107] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,107] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  34%|███▍      | 1841/5399 [00:43<01:10, 50.45 examples/s][2026-04-16 10:35:27,143] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,151] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,151] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,153] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:27,153] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:704] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:28,178] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:705] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:28,178] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:705] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:28,219] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:705] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:28,219] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:705] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  35%|███▌      | 1897/5399 [00:44<01:09, 50.67 examples/s][2026-04-16 10:35:28,254] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:705] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:29,236] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:706] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:29,236] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:706] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:29,279] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:706] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:29,279] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:706] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:29,283] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:706] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:29,283] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:706] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  36%|███▌      | 1953/5399 [00:45<01:07, 51.31 examples/s][2026-04-16 10:35:29,310] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:706] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:29,310] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:706] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:29,323] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:706] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:29,323] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:706] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:29,326] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:706] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:29,326] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:706] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:30,337] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:707] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:30,337] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:707] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:30,368] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:707] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:30,368] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:707] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:30,385] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:707] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:30,385] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:707] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  37%|███▋      | 2009/5399 [00:46<01:06, 51.28 examples/s][2026-04-16 10:35:30,397] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:707] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:30,398] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:707] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:30,412] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:707] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:30,412] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:707] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:31,463] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:708] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:31,495] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:708] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:31,495] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:708] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:31,496] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:708] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:31,506] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:708] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:31,506] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:708] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  38%|███▊      | 2066/5399 [00:47<01:05, 51.07 examples/s][2026-04-16 10:35:31,539] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:708] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:31,540] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:708] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,575] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,576] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,590] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,590] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,595] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,595] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,606] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,606] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,609] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,609] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,611] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,611] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,616] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,617] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  39%|███▉      | 2120/5399 [00:49<01:05, 50.21 examples/s][2026-04-16 10:35:32,662] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,662] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,670] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:32,670] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:709] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:33,675] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:710] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:33,675] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:710] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:33,728] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:710] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:33,728] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:710] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  40%|████      | 2176/5399 [00:50<01:03, 50.47 examples/s][2026-04-16 10:35:34,819] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:711] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:34,819] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:711] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:34,829] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:711] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:34,829] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:711] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:34,830] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:711] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:34,830] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:711] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:34,832] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:711] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:34,833] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:711] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:34,839] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:711] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:34,839] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:711] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:34,851] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:711] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:34,851] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:711] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  41%|████▏     | 2230/5399 [00:51<01:04, 49.45 examples/s][2026-04-16 10:35:35,901] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,901] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,906] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,906] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,917] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,917] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,937] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,937] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  42%|████▏     | 2289/5399 [00:52<01:00, 51.06 examples/s][2026-04-16 10:35:35,945] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,945] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,949] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,949] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,955] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,955] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,963] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,963] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,968] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,968] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,978] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:35,978] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:712] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:37,009] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:713] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:37,010] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:713] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:37,016] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:713] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:37,016] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:713] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:37,027] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:713] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:37,028] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:713] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:37,055] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:713] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:37,055] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:713] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  43%|████▎     | 2347/5399 [00:53<00:59, 51.27 examples/s][2026-04-16 10:35:38,106] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,106] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,115] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,115] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,117] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,117] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,143] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,143] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,155] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,155] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  44%|████▍     | 2400/5399 [00:54<00:59, 50.49 examples/s][2026-04-16 10:35:38,164] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,164] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,165] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,166] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:38,171] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:714] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:39,250] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:715] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  45%|████▌     | 2455/5399 [00:55<00:58, 50.45 examples/s][2026-04-16 10:35:39,250] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:715] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:39,255] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:715] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:39,285] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:715] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:39,285] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:715] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  47%|████▋     | 2513/5399 [00:56<00:56, 51.34 examples/s][2026-04-16 10:35:40,342] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:716] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:40,342] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:716] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:41,395] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:41,395] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:41,422] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:41,422] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:41,424] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:41,424] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:41,445] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:41,445] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:41,446] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:41,446] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  48%|████▊     | 2568/5399 [00:57<00:55, 50.68 examples/s][2026-04-16 10:35:41,468] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:41,481] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:41,481] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:717] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:42,443] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:718] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:42,464] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:718] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:42,479] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:718] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  49%|████▊     | 2626/5399 [00:58<00:53, 52.08 examples/s][2026-04-16 10:35:42,508] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:718] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:42,508] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:718] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:42,532] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:718] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:42,532] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:718] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:43,598] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:43,598] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:43,604] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:43,604] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:43,606] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:43,606] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:43,620] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:43,620] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:43,645] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  50%|████▉     | 2678/5399 [01:00<00:54, 49.98 examples/s][2026-04-16 10:35:43,674] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:43,674] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:43,691] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:43,691] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:719] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,700] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,700] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,733] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,733] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,752] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,753] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  51%|█████     | 2734/5399 [01:01<00:53, 50.14 examples/s][2026-04-16 10:35:44,761] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,778] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,778] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,785] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,785] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,794] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,794] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,798] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:44,798] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:720] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,809] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,809] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,822] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,831] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,831] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,833] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,833] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,837] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,838] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,841] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,841] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,851] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,851] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  52%|█████▏    | 2790/5399 [01:02<00:51, 50.18 examples/s][2026-04-16 10:35:45,881] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,881] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,883] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,896] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:45,896] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:721] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:46,950] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:722] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:46,951] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:722] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  53%|█████▎    | 2848/5399 [01:03<00:50, 50.31 examples/s][2026-04-16 10:35:48,111] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:723] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:48,111] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:723] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:48,141] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:723] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  54%|█████▍    | 2904/5399 [01:04<00:49, 50.09 examples/s][2026-04-16 10:35:48,153] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:723] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:48,153] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:723] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:48,158] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:723] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:48,158] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:723] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:48,178] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:723] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:48,178] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:723] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:49,186] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:724] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:49,186] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:724] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  55%|█████▍    | 2961/5399 [01:05<00:48, 50.65 examples/s][2026-04-16 10:35:49,264] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:724] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:49,264] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:724] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:49,266] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:724] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:49,266] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:724] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:50,293] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:725] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:50,293] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:725] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:50,299] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:725] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:50,299] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:725] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:50,308] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:725] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:50,329] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:725] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  56%|█████▌    | 3016/5399 [01:06<00:47, 50.33 examples/s][2026-04-16 10:35:50,347] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:725] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:50,347] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:725] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:50,357] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:725] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:50,368] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:725] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:50,368] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:725] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,395] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,395] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,400] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,400] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,405] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,405] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,419] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,419] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,444] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,444] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,446] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,448] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,448] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  57%|█████▋    | 3072/5399 [01:07<00:46, 50.48 examples/s][2026-04-16 10:35:51,459] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,459] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,461] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,461] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,484] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:51,484] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:726] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,484] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,484] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,487] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,487] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,498] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,498] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,531] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,532] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,535] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,535] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  58%|█████▊    | 3129/5399 [01:08<00:44, 50.88 examples/s][2026-04-16 10:35:52,557] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,557] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,564] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,564] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,581] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:52,581] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:727] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:53,584] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:728] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:53,585] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:728] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:53,609] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:728] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:53,609] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:728] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:53,627] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:728] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:53,632] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:728] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:53,633] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:728] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:53,634] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:728] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:53,634] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:728] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:53,640] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:728] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:53,640] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:728] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  59%|█████▉    | 3186/5399 [01:10<00:43, 51.20 examples/s][2026-04-16 10:35:54,659] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:729] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:54,659] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:729] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:54,696] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:729] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:54,696] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:729] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:54,714] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:729] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:54,714] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:729] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  60%|██████    | 3241/5399 [01:11<00:42, 51.28 examples/s][2026-04-16 10:35:54,733] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:729] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:55,765] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:730] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:55,765] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:730] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:55,787] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:730] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:55,787] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:730] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:55,796] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:730] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:55,796] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:730] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:55,802] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:730] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:55,819] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:730] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:55,823] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:730] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  61%|██████    | 3297/5399 [01:12<00:41, 50.82 examples/s][2026-04-16 10:35:56,930] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:731] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  62%|██████▏   | 3352/5399 [01:13<00:40, 50.73 examples/s][2026-04-16 10:35:56,937] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:731] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:56,937] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:731] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:56,942] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:731] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:56,943] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:731] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:56,947] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:731] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:56,947] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:731] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:57,980] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:732] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:57,998] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:732] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:57,998] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:732] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:58,008] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:732] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:58,008] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:732] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  63%|██████▎   | 3409/5399 [01:14<00:38, 51.21 examples/s][2026-04-16 10:35:58,028] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:732] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:58,028] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:732] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,050] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,050] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,053] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,053] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,064] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,064] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,070] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,081] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,099] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,099] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  64%|██████▍   | 3465/5399 [01:15<00:37, 51.34 examples/s][2026-04-16 10:35:59,104] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,113] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,113] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,126] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,137] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:35:59,137] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:733] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:00,158] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:734] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:00,161] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:734] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:00,199] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:734] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  65%|██████▌   | 3519/5399 [01:16<00:37, 50.70 examples/s][2026-04-16 10:36:00,207] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:734] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:00,207] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:734] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:00,209] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:734] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:00,209] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:734] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:00,216] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:734] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:00,216] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:734] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:00,235] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:734] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:00,235] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:734] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:01,286] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:735] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:01,292] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:735] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:01,307] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:735] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:01,307] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:735] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  66%|██████▋   | 3578/5399 [01:17<00:35, 51.04 examples/s][2026-04-16 10:36:01,342] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:735] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:02,416] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:736] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:02,416] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:736] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  67%|██████▋   | 3632/5399 [01:18<00:35, 50.49 examples/s][2026-04-16 10:36:02,453] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:736] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:02,453] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:736] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:02,479] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:736] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:02,479] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:736] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:03,492] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:737] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:03,492] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:737] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:03,504] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:737] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:03,504] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:737] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:03,519] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:737] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:03,519] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:737] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:03,521] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:737] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:03,521] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:737] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  68%|██████▊   | 3688/5399 [01:19<00:33, 50.62 examples/s][2026-04-16 10:36:03,545] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:737] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:03,545] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:737] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,610] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,611] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,625] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,625] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,631] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,631] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,633] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,645] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,645] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,663] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,663] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  69%|██████▉   | 3743/5399 [01:21<00:33, 49.98 examples/s][2026-04-16 10:36:04,693] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,693] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,701] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:04,701] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:738] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:05,716] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:739] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:05,716] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:739] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:05,724] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:739] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:05,724] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:739] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:05,743] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:739] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:05,743] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:739] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:05,745] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:739] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:05,745] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:739] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:05,762] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:739] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:05,773] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:739] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:05,773] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:739] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  70%|███████   | 3801/5399 [01:22<00:31, 50.73 examples/s][2026-04-16 10:36:05,786] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:739] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,794] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,794] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,802] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,802] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,830] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,830] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,836] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,836] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,843] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  71%|███████▏  | 3856/5399 [01:23<00:30, 50.77 examples/s][2026-04-16 10:36:06,862] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,862] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,868] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,868] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,873] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,876] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,889] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:06,889] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:740] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:07,907] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:07,907] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:07,946] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:07,970] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:07,970] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  72%|███████▏  | 3913/5399 [01:24<00:29, 50.80 examples/s][2026-04-16 10:36:07,976] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:07,976] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:07,989] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:07,989] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:07,994] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:07,994] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:07,998] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:07,998] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:741] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:09,015] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:742] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:09,015] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:742] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:09,030] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:742] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:09,030] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:742] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:09,049] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:742] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:09,049] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:742] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:09,060] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:742] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  73%|███████▎  | 3968/5399 [01:25<00:28, 50.64 examples/s][2026-04-16 10:36:09,098] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:742] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:09,098] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:742] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,119] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,119] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,132] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,132] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  75%|███████▍  | 4026/5399 [01:26<00:26, 51.33 examples/s][2026-04-16 10:36:10,165] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,165] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,178] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,178] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,179] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,179] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,187] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,187] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,200] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:10,200] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:743] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:11,266] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:744] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:11,268] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:744] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  76%|███████▌  | 4082/5399 [01:27<00:25, 51.13 examples/s][2026-04-16 10:36:12,309] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:745] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:12,309] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:745] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  77%|███████▋  | 4135/5399 [01:28<00:25, 50.35 examples/s][2026-04-16 10:36:12,369] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:745] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:12,369] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:745] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:12,376] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:745] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:12,376] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:745] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:13,470] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:746] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:13,471] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:746] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:13,513] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:746] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:13,513] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:746] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  78%|███████▊  | 4192/5399 [01:29<00:24, 49.97 examples/s][2026-04-16 10:36:13,527] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:746] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:13,527] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:746] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:13,537] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:746] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:13,537] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:746] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:14,586] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:14,586] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:14,592] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:14,592] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:14,611] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:14,611] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:14,620] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:14,620] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  79%|███████▊  | 4249/5399 [01:31<00:22, 50.57 examples/s][2026-04-16 10:36:14,630] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:14,630] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:14,637] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:14,637] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:14,646] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:747] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:15,680] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:748] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:15,681] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:748] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:15,702] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:748] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:15,703] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:748] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:15,727] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:748] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:15,727] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:748] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:15,730] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:748] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:15,739] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:748] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:15,739] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:748] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  80%|███████▉  | 4304/5399 [01:32<00:21, 50.14 examples/s][2026-04-16 10:36:15,745] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:748] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:15,745] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:748] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:15,758] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:748] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  81%|████████  | 4359/5399 [01:33<00:20, 50.19 examples/s][2026-04-16 10:36:16,840] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:749] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:16,872] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:749] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:16,872] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:749] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  82%|████████▏ | 4417/5399 [01:34<00:19, 50.31 examples/s][2026-04-16 10:36:17,988] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:750] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:17,988] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:750] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:17,992] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:750] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:17,992] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:750] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:19,049] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:751] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:19,050] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:751] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:19,056] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:751] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:19,056] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:751] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:19,062] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:751] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:19,062] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:751] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:19,067] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:751] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:19,067] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:751] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:19,081] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:751] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  83%|████████▎ | 4474/5399 [01:35<00:18, 50.76 examples/s][2026-04-16 10:36:20,130] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:752] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:20,131] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:752] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:20,145] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:752] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:20,145] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:752] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:20,160] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:752] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:20,160] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:752] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:20,164] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:752] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:20,171] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:752] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:20,172] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:752] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  84%|████████▍ | 4529/5399 [01:36<00:17, 50.51 examples/s][2026-04-16 10:36:21,206] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:753] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:21,207] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:753] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:21,215] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:753] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:21,215] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:753] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:21,229] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:753] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:21,248] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:753] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:21,248] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:753] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:21,253] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:753] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:21,253] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:753] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  85%|████████▍ | 4585/5399 [01:37<00:16, 50.72 examples/s][2026-04-16 10:36:21,288] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:753] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,350] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,351] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,355] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,355] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,357] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,357] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,368] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,374] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,374] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  86%|████████▌ | 4642/5399 [01:38<00:14, 50.66 examples/s][2026-04-16 10:36:22,407] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,407] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,414] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,437] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:22,437] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:754] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:23,492] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:755] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:23,495] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:755] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:23,496] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:755] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  87%|████████▋ | 4698/5399 [01:39<00:13, 50.54 examples/s][2026-04-16 10:36:23,527] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:755] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:23,544] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:755] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,541] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,542] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,550] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,550] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,570] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,570] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,572] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,572] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  88%|████████▊ | 4748/5399 [01:40<00:13, 49.28 examples/s][2026-04-16 10:36:24,600] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,600] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,612] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,613] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,616] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,616] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,619] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,620] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,622] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,622] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,633] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,633] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:24,644] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:756] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:25,698] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:757] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:25,698] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:757] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:25,729] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:757] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:25,729] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:757] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  89%|████████▉ | 4808/5399 [01:42<00:11, 50.28 examples/s][2026-04-16 10:36:25,746] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:757] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:25,746] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:757] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:25,775] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:757] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:26,729] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:758] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:26,730] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:758] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:26,768] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:758] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:26,771] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:758] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:26,771] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:758] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  90%|█████████ | 4868/5399 [01:43<00:10, 52.20 examples/s][2026-04-16 10:36:26,802] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:758] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:26,802] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:758] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:26,813] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:758] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:26,813] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:758] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:27,915] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:759] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:27,915] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:759] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  91%|█████████ | 4920/5399 [01:44<00:09, 50.26 examples/s][2026-04-16 10:36:27,936] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:759] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:27,936] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:759] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:27,941] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:759] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:28,977] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:760] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:28,993] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:760] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:28,993] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:760] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:29,002] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:760] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:29,002] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:760] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:29,004] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:760] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:29,022] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:760] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  92%|█████████▏| 4976/5399 [01:45<00:08, 50.51 examples/s][2026-04-16 10:36:29,045] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:760] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:29,057] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:760] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:29,057] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:760] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:30,075] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:761] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:30,075] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:761] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:30,097] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:761] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:30,097] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:761] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:30,099] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:761] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:30,099] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:761] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:30,109] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:761] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:30,109] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:761] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  93%|█████████▎| 5034/5399 [01:46<00:07, 51.15 examples/s][2026-04-16 10:36:30,134] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:761] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:30,134] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:761] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,176] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,176] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,183] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,184] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,190] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,190] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,202] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,202] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,204] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,204] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,221] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,221] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  94%|█████████▍| 5088/5399 [01:47<00:06, 50.55 examples/s][2026-04-16 10:36:31,240] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,240] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,259] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:31,259] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:762] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  95%|█████████▍| 5119/5399 [01:47<00:04, 60.22 examples/s][2026-04-16 10:36:32,265] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:763] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:32,265] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:763] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:32,267] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:763] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  95%|█████████▌| 5148/5399 [01:48<00:04, 50.26 examples/s][2026-04-16 10:36:32,278] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:763] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:32,279] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:763] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:33,350] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:764] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:33,376] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:764] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:33,376] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:764] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:33,380] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:764] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:33,380] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:764] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:33,388] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:764] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:33,394] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:764] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:33,394] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:764] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  96%|█████████▋| 5201/5399 [01:49<00:04, 48.69 examples/s][2026-04-16 10:36:34,463] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:765] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:34,464] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:765] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:34,467] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:765] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:34,474] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:765] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:34,474] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:765] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  97%|█████████▋| 5257/5399 [01:50<00:02, 49.54 examples/s][2026-04-16 10:36:34,521] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:765] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:34,521] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:765] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:34,537] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:765] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:34,537] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:765] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,582] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,582] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,590] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,590] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,596] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,596] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,614] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,614] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,616] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,616] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,621] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,621] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,643] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  98%|█████████▊| 5312/5399 [01:52<00:01, 49.35 examples/s][2026-04-16 10:36:35,643] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,653] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,653] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,657] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,657] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,669] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:35,669] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:766] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:36,697] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:767] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:36,698] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:767] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:36,722] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:767] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:36,726] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:767] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:36,726] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:767] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:36,729] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:767] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96):  99%|█████████▉| 5367/5399 [01:53<00:00, 49.65 examples/s][2026-04-16 10:36:36,759] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:767] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:36,773] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:767] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:36,773] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:767] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing train dataset (num_proc=96): 100%|██████████| 5399/5399 [01:54<00:00, 47.24 examples/s]

Adding EOS to eval dataset (num_proc=96):   0%|          | 0/285 [00:00<?, ? examples/s]
Adding EOS to eval dataset (num_proc=96):   1%|          | 3/285 [00:06<10:22,  2.21s/ examples]
Adding EOS to eval dataset (num_proc=96):  15%|█▍        | 42/285 [00:06<00:28,  8.68 examples/s]
Adding EOS to eval dataset (num_proc=96):  79%|███████▉  | 225/285 [00:06<00:00, 61.99 examples/s]
Adding EOS to eval dataset (num_proc=96): 100%|██████████| 285/285 [00:08<00:00, 34.68 examples/s]

Tokenizing eval dataset (num_proc=96):   0%|          | 0/285 [00:00<?, ? examples/s][2026-04-16 10:36:55,837] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1460] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:36:55,837] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1460] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):   1%|          | 3/285 [00:08<12:53,  2.74s/ examples][2026-04-16 10:36:56,938] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1461] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):   2%|▏         | 6/285 [00:09<06:16,  1.35s/ examples]
Tokenizing eval dataset (num_proc=96):   3%|▎         | 9/285 [00:10<04:08,  1.11 examples/s]
Tokenizing eval dataset (num_proc=96):   4%|▍         | 12/285 [00:11<03:08,  1.45 examples/s]
Tokenizing eval dataset (num_proc=96):   5%|▌         | 15/285 [00:12<02:35,  1.74 examples/s]
Tokenizing eval dataset (num_proc=96):   6%|▋         | 18/285 [00:14<02:24,  1.85 examples/s]
Tokenizing eval dataset (num_proc=96):   7%|▋         | 21/285 [00:15<02:07,  2.07 examples/s]
Tokenizing eval dataset (num_proc=96):   8%|▊         | 24/285 [00:16<01:56,  2.23 examples/s]
Tokenizing eval dataset (num_proc=96):   9%|▉         | 27/285 [00:17<01:49,  2.36 examples/s]
Tokenizing eval dataset (num_proc=96):  11%|█         | 30/285 [00:18<01:42,  2.49 examples/s][2026-04-16 10:37:07,172] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1470] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  12%|█▏        | 33/285 [00:19<01:38,  2.56 examples/s]
Tokenizing eval dataset (num_proc=96):  13%|█▎        | 36/285 [00:20<01:36,  2.59 examples/s][2026-04-16 10:37:09,405] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1472] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:09,405] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1472] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  14%|█▎        | 39/285 [00:21<01:33,  2.63 examples/s]
Tokenizing eval dataset (num_proc=96):  15%|█▍        | 42/285 [00:22<01:31,  2.66 examples/s]
Tokenizing eval dataset (num_proc=96):  16%|█▌        | 45/285 [00:23<01:29,  2.68 examples/s][2026-04-16 10:37:12,701] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1475] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:12,701] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1475] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  17%|█▋        | 48/285 [00:25<01:27,  2.69 examples/s]
Tokenizing eval dataset (num_proc=96):  18%|█▊        | 51/285 [00:26<01:26,  2.71 examples/s][2026-04-16 10:37:14,905] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1477] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  19%|█▉        | 54/285 [00:27<01:25,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  20%|██        | 57/285 [00:28<01:24,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  21%|██        | 60/285 [00:29<01:22,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  22%|██▏       | 63/285 [00:30<01:22,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  23%|██▎       | 66/285 [00:31<01:20,  2.73 examples/s][2026-04-16 10:37:20,446] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1482] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:20,446] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1482] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  24%|██▍       | 69/285 [00:32<01:19,  2.70 examples/s][2026-04-16 10:37:21,529] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1483] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  25%|██▌       | 72/285 [00:33<01:18,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  26%|██▋       | 75/285 [00:35<01:17,  2.71 examples/s][2026-04-16 10:37:23,734] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1485] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  27%|██▋       | 78/285 [00:36<01:16,  2.72 examples/s][2026-04-16 10:37:24,841] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1486] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:24,841] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1486] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  28%|██▊       | 81/285 [00:37<01:14,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  29%|██▉       | 84/285 [00:38<01:14,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  31%|███       | 87/285 [00:39<01:12,  2.72 examples/s][2026-04-16 10:37:28,151] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1489] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:28,151] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1489] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  32%|███▏      | 90/285 [00:40<01:11,  2.72 examples/s][2026-04-16 10:37:29,245] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1490] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  33%|███▎      | 93/285 [00:41<01:10,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  34%|███▎      | 96/285 [00:42<01:09,  2.73 examples/s]
Tokenizing eval dataset (num_proc=96):  35%|███▍      | 99/285 [00:43<01:08,  2.73 examples/s]
Tokenizing eval dataset (num_proc=96):  36%|███▌      | 102/285 [00:44<01:07,  2.72 examples/s][2026-04-16 10:37:33,636] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1494] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:33,636] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1494] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  37%|███▋      | 105/285 [00:46<01:05,  2.74 examples/s][2026-04-16 10:37:34,757] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1495] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:34,757] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1495] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  38%|███▊      | 108/285 [00:47<01:05,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  39%|███▉      | 111/285 [00:48<01:04,  2.72 examples/s][2026-04-16 10:37:36,969] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1497] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:36,969] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1497] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  40%|████      | 114/285 [00:49<01:03,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  41%|████      | 117/285 [00:50<01:01,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  42%|████▏     | 120/285 [00:51<01:00,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  43%|████▎     | 123/285 [00:52<00:59,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  44%|████▍     | 126/285 [00:53<00:58,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  45%|████▌     | 129/285 [00:54<00:57,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  46%|████▋     | 132/285 [00:56<00:56,  2.71 examples/s][2026-04-16 10:37:44,694] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1504] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:44,694] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1504] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  47%|████▋     | 135/285 [00:57<00:55,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  48%|████▊     | 138/285 [00:58<00:54,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  49%|████▉     | 141/285 [00:59<00:53,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  51%|█████     | 144/285 [01:00<00:52,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  52%|█████▏    | 147/285 [01:01<00:50,  2.72 examples/s][2026-04-16 10:37:50,233] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1509] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:50,233] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1509] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  53%|█████▎    | 150/285 [01:02<00:49,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  54%|█████▎    | 153/285 [01:03<00:48,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  55%|█████▍    | 156/285 [01:04<00:47,  2.73 examples/s]
Tokenizing eval dataset (num_proc=96):  56%|█████▌    | 159/285 [01:05<00:46,  2.70 examples/s]
Tokenizing eval dataset (num_proc=96):  57%|█████▋    | 162/285 [01:07<00:45,  2.70 examples/s]
Tokenizing eval dataset (num_proc=96):  58%|█████▊    | 165/285 [01:08<00:44,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  59%|█████▉    | 168/285 [01:09<00:43,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  60%|██████    | 171/285 [01:10<00:42,  2.71 examples/s][2026-04-16 10:37:59,092] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1517] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:59,092] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1517] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:59,098] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1517] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:37:59,099] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1517] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  61%|██████    | 174/285 [01:11<00:41,  2.70 examples/s]
Tokenizing eval dataset (num_proc=96):  62%|██████▏   | 177/285 [01:12<00:40,  2.70 examples/s][2026-04-16 10:38:01,343] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1519] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:38:01,343] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1519] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  63%|██████▎   | 180/285 [01:13<00:39,  2.69 examples/s]
Tokenizing eval dataset (num_proc=96):  64%|██████▍   | 183/285 [01:14<00:37,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  65%|██████▌   | 186/285 [01:15<00:36,  2.69 examples/s]
Tokenizing eval dataset (num_proc=96):  66%|██████▋   | 189/285 [01:17<00:35,  2.69 examples/s]
Tokenizing eval dataset (num_proc=96):  67%|██████▋   | 192/285 [01:18<00:34,  2.70 examples/s][2026-04-16 10:38:06,869] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1524] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:38:06,870] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1524] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  68%|██████▊   | 195/285 [01:19<00:33,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  69%|██████▉   | 198/285 [01:20<00:32,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  71%|███████   | 201/285 [01:21<00:30,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  72%|███████▏  | 204/285 [01:22<00:29,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  73%|███████▎  | 207/285 [01:23<00:28,  2.70 examples/s]
Tokenizing eval dataset (num_proc=96):  74%|███████▎  | 210/285 [01:24<00:27,  2.72 examples/s][2026-04-16 10:38:13,504] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1530] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:38:13,505] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1530] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  75%|███████▍  | 213/285 [01:25<00:26,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  76%|███████▌  | 216/285 [01:27<00:25,  2.70 examples/s]
Tokenizing eval dataset (num_proc=96):  77%|███████▋  | 219/285 [01:28<00:24,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  78%|███████▊  | 222/285 [01:29<00:23,  2.72 examples/s][2026-04-16 10:38:17,888] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1534] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:38:17,889] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1534] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  79%|███████▉  | 225/285 [01:30<00:21,  2.73 examples/s][2026-04-16 10:38:19,014] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1535] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:38:19,014] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1535] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  80%|████████  | 228/285 [01:31<00:20,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  81%|████████  | 231/285 [01:32<00:19,  2.73 examples/s]
Tokenizing eval dataset (num_proc=96):  82%|████████▏ | 234/285 [01:33<00:18,  2.74 examples/s][2026-04-16 10:38:22,308] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1538] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:38:22,308] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1538] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  83%|████████▎ | 237/285 [01:34<00:17,  2.73 examples/s][2026-04-16 10:38:23,406] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1539] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:38:23,407] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1539] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  84%|████████▍ | 240/285 [01:35<00:16,  2.73 examples/s]
Tokenizing eval dataset (num_proc=96):  85%|████████▌ | 243/285 [01:36<00:15,  2.73 examples/s][2026-04-16 10:38:25,598] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1541] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:38:25,598] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1541] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  86%|████████▋ | 246/285 [01:37<00:14,  2.74 examples/s]
Tokenizing eval dataset (num_proc=96):  87%|████████▋ | 249/285 [01:39<00:13,  2.73 examples/s]
Tokenizing eval dataset (num_proc=96):  88%|████████▊ | 252/285 [01:40<00:12,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  89%|████████▉ | 255/285 [01:41<00:11,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  91%|█████████ | 258/285 [01:42<00:09,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  92%|█████████▏| 261/285 [01:43<00:08,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  93%|█████████▎| 264/285 [01:44<00:07,  2.71 examples/s]
Tokenizing eval dataset (num_proc=96):  94%|█████████▎| 267/285 [01:45<00:06,  2.72 examples/s][2026-04-16 10:38:34,430] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1549] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:38:34,431] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1549] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  95%|█████████▍| 270/285 [01:46<00:05,  2.70 examples/s]
Tokenizing eval dataset (num_proc=96):  96%|█████████▌| 273/285 [01:47<00:04,  2.71 examples/s][2026-04-16 10:38:36,648] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1551] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:38:36,649] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1551] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:38:36,654] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:929] [PID:1551] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+chosen. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.
[2026-04-16 10:38:36,654] [WARNING] [trl.trainer.dpo_trainer.tokenize_fn:935] [PID:1551] [RANK 0] Mismatch between tokenized prompt and the start of tokenized prompt+rejected. This may be due to unexpected tokenizer behavior, whitespace issues, or special token handling. Verify that the tokenizer is processing text consistently.

Tokenizing eval dataset (num_proc=96):  97%|█████████▋| 276/285 [01:49<00:03,  2.70 examples/s]
Tokenizing eval dataset (num_proc=96):  98%|█████████▊| 279/285 [01:50<00:02,  2.72 examples/s]
Tokenizing eval dataset (num_proc=96):  99%|█████████▊| 281/285 [01:51<00:01,  2.45 examples/s]
Tokenizing eval dataset (num_proc=96):  99%|█████████▉| 283/285 [01:52<00:00,  2.26 examples/s]
Tokenizing eval dataset (num_proc=96): 100%|██████████| 285/285 [01:53<00:00,  2.13 examples/s]
Tokenizing eval dataset (num_proc=96): 100%|██████████| 285/285 [01:54<00:00,  2.49 examples/s]
[2026-04-16 10:38:46,976] [INFO] [axolotl.train.save_initial_configs:452] [PID:135] Pre-saving tokenizer to kikiyaa/qwen-dpo-finetuned-ver2...
[2026-04-16 10:38:47,137] [INFO] [axolotl.train.save_initial_configs:457] [PID:135] Pre-saving model config to kikiyaa/qwen-dpo-finetuned-ver2...
[2026-04-16 10:38:47,145] [INFO] [axolotl.train.execute_training:226] [PID:135] Starting trainer...

  0%|          | 0/1013 [00:00<?, ?it/s]

  0%|          | 0/143 [00:00<?, ?it/s]

  1%|▏         | 2/143 [00:00<00:20,  6.77it/s]

  2%|▏         | 3/143 [00:00<00:28,  4.88it/s]

  3%|▎         | 4/143 [00:00<00:30,  4.55it/s]

  3%|▎         | 5/143 [00:01<00:33,  4.14it/s]

  4%|▍         | 6/143 [00:01<00:32,  4.22it/s]

  5%|▍         | 7/143 [00:01<00:30,  4.47it/s]

  6%|▌         | 8/143 [00:01<00:32,  4.19it/s]

  6%|▋         | 9/143 [00:01<00:29,  4.47it/s]

  7%|▋         | 10/143 [00:02<00:30,  4.34it/s]

  8%|▊         | 11/143 [00:02<00:30,  4.36it/s]

  8%|▊         | 12/143 [00:02<00:32,  4.07it/s]

  9%|▉         | 13/143 [00:03<00:32,  3.98it/s]

 10%|▉         | 14/143 [00:03<00:34,  3.76it/s]

 10%|█         | 15/143 [00:03<00:34,  3.76it/s]

 11%|█         | 16/143 [00:03<00:30,  4.10it/s]

 12%|█▏        | 17/143 [00:03<00:28,  4.39it/s]

 13%|█▎        | 18/143 [00:04<00:27,  4.47it/s]

 13%|█▎        | 19/143 [00:04<00:31,  3.98it/s]

 14%|█▍        | 20/143 [00:04<00:32,  3.76it/s]

 15%|█▍        | 21/143 [00:05<00:31,  3.85it/s]

 15%|█▌        | 22/143 [00:05<00:31,  3.80it/s]

 16%|█▌        | 23/143 [00:05<00:30,  3.88it/s]

 17%|█▋        | 24/143 [00:05<00:31,  3.81it/s]

 17%|█▋        | 25/143 [00:06<00:29,  3.98it/s]

 18%|█▊        | 26/143 [00:06<00:28,  4.15it/s]

 19%|█▉        | 27/143 [00:06<00:31,  3.69it/s]

 20%|█▉        | 28/143 [00:06<00:32,  3.57it/s]

 20%|██        | 29/143 [00:07<00:31,  3.59it/s]

 21%|██        | 30/143 [00:07<00:32,  3.52it/s]

 22%|██▏       | 31/143 [00:07<00:31,  3.57it/s]

 22%|██▏       | 32/143 [00:07<00:29,  3.78it/s]

 23%|██▎       | 33/143 [00:08<00:27,  4.01it/s]

 24%|██▍       | 34/143 [00:08<00:27,  4.04it/s]

 24%|██▍       | 35/143 [00:08<00:27,  3.91it/s]

 25%|██▌       | 36/143 [00:08<00:26,  4.11it/s]

 26%|██▌       | 37/143 [00:09<00:25,  4.13it/s]

 27%|██▋       | 38/143 [00:09<00:25,  4.11it/s]

 27%|██▋       | 39/143 [00:09<00:27,  3.84it/s]

 28%|██▊       | 40/143 [00:09<00:25,  4.04it/s]

 29%|██▊       | 41/143 [00:10<00:25,  4.06it/s]

 29%|██▉       | 42/143 [00:10<00:26,  3.81it/s]

 30%|███       | 43/143 [00:10<00:26,  3.74it/s]

 31%|███       | 44/143 [00:11<00:25,  3.82it/s]

 31%|███▏      | 45/143 [00:11<00:24,  3.99it/s]

 32%|███▏      | 46/143 [00:11<00:30,  3.20it/s]

 33%|███▎      | 47/143 [00:11<00:29,  3.28it/s]

 34%|███▎      | 48/143 [00:12<00:27,  3.47it/s]

 34%|███▍      | 49/143 [00:12<00:26,  3.52it/s]

 35%|███▍      | 50/143 [00:12<00:26,  3.57it/s]

 36%|███▌      | 51/143 [00:13<00:25,  3.60it/s]

 36%|███▋      | 52/143 [00:13<00:24,  3.72it/s]

 37%|███▋      | 53/143 [00:13<00:23,  3.90it/s]

 38%|███▊      | 54/143 [00:13<00:24,  3.64it/s]

 38%|███▊      | 55/143 [00:14<00:23,  3.75it/s]

 39%|███▉      | 56/143 [00:14<00:22,  3.94it/s]

 40%|███▉      | 57/143 [00:14<00:21,  3.98it/s]

 41%|████      | 58/143 [00:14<00:19,  4.28it/s]

 41%|████▏     | 59/143 [00:14<00:19,  4.38it/s]

 42%|████▏     | 60/143 [00:15<00:20,  4.15it/s]

 43%|████▎     | 61/143 [00:15<00:18,  4.40it/s]

 43%|████▎     | 62/143 [00:15<00:21,  3.74it/s]

 44%|████▍     | 63/143 [00:16<00:21,  3.76it/s]

 45%|████▍     | 64/143 [00:16<00:22,  3.54it/s]

 45%|████▌     | 65/143 [00:16<00:21,  3.55it/s]

 46%|████▌     | 66/143 [00:17<00:23,  3.24it/s]

 47%|████▋     | 67/143 [00:17<00:22,  3.42it/s]

 48%|████▊     | 68/143 [00:17<00:21,  3.52it/s]

 48%|████▊     | 69/143 [00:17<00:23,  3.18it/s]

 49%|████▉     | 70/143 [00:18<00:20,  3.51it/s]

 50%|████▉     | 71/143 [00:18<00:20,  3.56it/s]

 50%|█████     | 72/143 [00:18<00:19,  3.59it/s]

 51%|█████     | 73/143 [00:18<00:19,  3.54it/s]

 52%|█████▏    | 74/143 [00:19<00:19,  3.53it/s]

 52%|█████▏    | 75/143 [00:19<00:19,  3.47it/s]

 53%|█████▎    | 76/143 [00:19<00:18,  3.61it/s]

 54%|█████▍    | 77/143 [00:20<00:17,  3.75it/s]

 55%|█████▍    | 78/143 [00:20<00:15,  4.07it/s]

 55%|█████▌    | 79/143 [00:20<00:15,  4.08it/s]

 56%|█████▌    | 80/143 [00:20<00:18,  3.49it/s]

 57%|█████▋    | 81/143 [00:21<00:16,  3.84it/s]

 57%|█████▋    | 82/143 [00:21<00:17,  3.50it/s]

 58%|█████▊    | 83/143 [00:21<00:17,  3.38it/s]

 59%|█████▊    | 84/143 [00:22<00:17,  3.39it/s]

 59%|█████▉    | 85/143 [00:22<00:16,  3.57it/s]

 60%|██████    | 86/143 [00:22<00:13,  4.13it/s]

 61%|██████    | 87/143 [00:22<00:13,  4.23it/s]

 62%|██████▏   | 88/143 [00:22<00:13,  4.08it/s]

 62%|██████▏   | 89/143 [00:23<00:13,  3.93it/s]

 63%|██████▎   | 90/143 [00:23<00:13,  3.96it/s]

 64%|██████▎   | 91/143 [00:23<00:12,  4.26it/s]

 64%|██████▍   | 92/143 [00:23<00:12,  4.11it/s]

 65%|██████▌   | 93/143 [00:24<00:12,  4.10it/s]

 66%|██████▌   | 94/143 [00:24<00:12,  3.80it/s]

 66%|██████▋   | 95/143 [00:24<00:14,  3.40it/s]

 67%|██████▋   | 96/143 [00:25<00:12,  3.63it/s]

 68%|██████▊   | 97/143 [00:25<00:11,  3.99it/s]

 69%|██████▊   | 98/143 [00:25<00:10,  4.12it/s]

 69%|██████▉   | 99/143 [00:25<00:11,  3.90it/s]

 70%|██████▉   | 100/143 [00:26<00:11,  3.63it/s]

 71%|███████   | 101/143 [00:26<00:10,  3.88it/s]

 71%|███████▏  | 102/143 [00:26<00:10,  3.82it/s]

 72%|███████▏  | 103/143 [00:26<00:10,  3.65it/s]

 73%|███████▎  | 104/143 [00:27<00:09,  3.90it/s]

 73%|███████▎  | 105/143 [00:27<00:09,  3.82it/s]

 74%|███████▍  | 106/143 [00:27<00:09,  3.77it/s]

 75%|███████▍  | 107/143 [00:27<00:09,  3.94it/s]

 76%|███████▌  | 108/143 [00:28<00:08,  4.08it/s]

 76%|███████▌  | 109/143 [00:28<00:08,  4.08it/s]

 77%|███████▋  | 110/143 [00:28<00:07,  4.18it/s]

 78%|███████▊  | 111/143 [00:28<00:08,  3.79it/s]

 78%|███████▊  | 112/143 [00:29<00:07,  3.94it/s]

 79%|███████▉  | 113/143 [00:29<00:08,  3.44it/s]

 80%|███████▉  | 114/143 [00:29<00:08,  3.61it/s]

 80%|████████  | 115/143 [00:30<00:07,  3.60it/s]

 81%|████████  | 116/143 [00:30<00:07,  3.80it/s]

 82%|████████▏ | 117/143 [00:30<00:07,  3.64it/s]

 83%|████████▎ | 118/143 [00:30<00:06,  3.99it/s]

 83%|████████▎ | 119/143 [00:30<00:05,  4.28it/s]

 84%|████████▍ | 120/143 [00:31<00:05,  4.06it/s]

 85%|████████▍ | 121/143 [00:31<00:05,  3.87it/s]

 85%|████████▌ | 122/143 [00:31<00:04,  4.32it/s]

 86%|████████▌ | 123/143 [00:31<00:04,  4.04it/s]

 87%|████████▋ | 124/143 [00:32<00:04,  4.14it/s]

 87%|████████▋ | 125/143 [00:32<00:03,  4.58it/s]

 88%|████████▊ | 126/143 [00:32<00:03,  4.53it/s]

 89%|████████▉ | 127/143 [00:32<00:03,  4.18it/s]

 90%|████████▉ | 128/143 [00:33<00:03,  3.95it/s]

 90%|█████████ | 129/143 [00:33<00:03,  4.07it/s]

 91%|█████████ | 130/143 [00:33<00:03,  4.21it/s]

 92%|█████████▏| 131/143 [00:33<00:02,  4.32it/s]

 92%|█████████▏| 132/143 [00:34<00:02,  4.24it/s]

 93%|█████████▎| 133/143 [00:34<00:02,  3.69it/s]

 94%|█████████▎| 134/143 [00:34<00:02,  3.67it/s]

 94%|█████████▍| 135/143 [00:34<00:02,  3.54it/s]

 95%|█████████▌| 136/143 [00:35<00:01,  3.75it/s]

 96%|█████████▌| 137/143 [00:35<00:01,  3.59it/s]

 97%|█████████▋| 138/143 [00:35<00:01,  3.58it/s]

 97%|█████████▋| 139/143 [00:36<00:01,  3.71it/s]

 98%|█████████▊| 140/143 [00:36<00:00,  4.05it/s]

 99%|█████████▊| 141/143 [00:36<00:00,  3.96it/s]

 99%|█████████▉| 142/143 [00:36<00:00,  3.85it/s]

100%|██████████| 143/143 [00:37<00:00,  3.83it/s]
                                        

                                                 
{'eval_loss': '0.6931', 'eval_runtime': '39.08', 'eval_samples_per_second': '7.293', 'eval_steps_per_second': '3.659', 'eval_entropy': '1.729', 'eval_num_tokens': '0', 'eval_logits/chosen': '0.8636', 'eval_logits/rejected': '-0.6579', 'eval_mean_token_accuracy': '0.4934', 'eval_rewards/chosen': '0', 'eval_rewards/rejected': '0', 'eval_rewards/accuracies': '0', 'eval_rewards/margins': '0', 'eval_logps/chosen': '-111.2', 'eval_logps/rejected': '-175', 'epoch': 0}

  0%|          | 0/1013 [00:39<?, ?it/s]

100%|██████████| 143/143 [00:37<00:00,  3.83it/s]

                                                 
  0%|          | 1/1013 [00:47<13:22:13, 47.56s/it]
                                                   
{'loss': '0.6931', 'grad_norm': '83.5', 'learning_rate': '0', 'entropy': '1.75', 'num_tokens': '1.017e+04', 'logits/chosen': '1.082', 'logits/rejected': '-0.5941', 'mean_token_accuracy': '0.5048', 'rewards/chosen': '0', 'rewards/rejected': '0', 'rewards/accuracies': '0', 'rewards/margins': '0', 'logps/chosen': '-110.6', 'logps/rejected': '-182.5', 'epoch': '0.002963'}

  0%|          | 1/1013 [00:47<13:22:13, 47.56s/it]
  0%|          | 2/1013 [00:54<6:35:36, 23.48s/it] 
  0%|          | 3/1013 [01:01<4:28:53, 15.97s/it]
  0%|          | 4/1013 [01:08<3:31:38, 12.59s/it]
  0%|          | 5/1013 [01:14<2:53:31, 10.33s/it]
  1%|          | 6/1013 [01:22<2:34:58,  9.23s/it]
  1%|          | 7/1013 [01:28<2:20:10,  8.36s/it]
  1%|          | 8/1013 [01:35<2:12:30,  7.91s/it]
  1%|          | 9/1013 [01:42<2:06:06,  7.54s/it]
  1%|          | 10/1013 [01:49<2:02:38,  7.34s/it]
                                                   
{'loss': '0.5316', 'grad_norm': '29.88', 'learning_rate': '4.455e-06', 'entropy': '1.73', 'num_tokens': '8.912e+04', 'logits/chosen': '0.8213', 'logits/rejected': '-0.6694', 'mean_token_accuracy': '0.5101', 'rewards/chosen': '0.3062', 'rewards/rejected': '-0.1129', 'rewards/accuracies': '0.8125', 'rewards/margins': '0.4191', 'logps/chosen': '-100.8', 'logps/rejected': '-171.8', 'epoch': '0.02963'}

  1%|          | 10/1013 [01:49<2:02:38,  7.34s/it]
  1%|          | 11/1013 [01:56<2:02:58,  7.36s/it]
  1%|          | 12/1013 [02:03<1:59:21,  7.15s/it]
  1%|▏         | 13/1013 [02:10<2:00:35,  7.24s/it]
  1%|▏         | 14/1013 [02:17<1:58:43,  7.13s/it]
  1%|▏         | 15/1013 [02:25<2:00:13,  7.23s/it]
  2%|▏         | 16/1013 [02:32<2:01:38,  7.32s/it]
  2%|▏         | 17/1013 [02:39<2:00:19,  7.25s/it]
  2%|▏         | 18/1013 [02:46<1:59:47,  7.22s/it]
  2%|▏         | 19/1013 [02:53<1:59:24,  7.21s/it]
  2%|▏         | 20/1013 [03:01<2:00:00,  7.25s/it]
                                                   
{'loss': '0.03662', 'grad_norm': '0.07471', 'learning_rate': '9.406e-06', 'entropy': '1.585', 'num_tokens': '1.828e+05', 'logits/chosen': '1.218', 'logits/rejected': '-0.8403', 'mean_token_accuracy': '0.5997', 'rewards/chosen': '2.911', 'rewards/rejected': '-2.458', 'rewards/accuracies': '1', 'rewards/margins': '5.369', 'logps/chosen': '-77.17', 'logps/rejected': '-196.3', 'epoch': '0.05926'}

  2%|▏         | 20/1013 [03:01<2:00:00,  7.25s/it]
  2%|▏         | 21/1013 [03:08<1:57:36,  7.11s/it]
  2%|▏         | 22/1013 [03:15<1:57:37,  7.12s/it]
  2%|▏         | 23/1013 [03:22<1:58:26,  7.18s/it]
  2%|▏         | 24/1013 [03:29<1:56:28,  7.07s/it]
  2%|▏         | 25/1013 [03:36<1:57:33,  7.14s/it]
  3%|▎         | 26/1013 [03:43<1:56:58,  7.11s/it]
  3%|▎         | 27/1013 [03:51<1:58:01,  7.18s/it]
  3%|▎         | 28/1013 [03:58<1:57:11,  7.14s/it]
  3%|▎         | 29/1013 [04:05<1:57:01,  7.14s/it]
  3%|▎         | 30/1013 [04:12<1:55:47,  7.07s/it]
                                                   
{'loss': '6.737e-05', 'grad_norm': '0.01929', 'learning_rate': '1.436e-05', 'entropy': '1.412', 'num_tokens': '2.748e+05', 'logits/chosen': '2.265', 'logits/rejected': '-1.035', 'mean_token_accuracy': '0.6582', 'rewards/chosen': '4.463', 'rewards/rejected': '-8.481', 'rewards/accuracies': '1', 'rewards/margins': '12.94', 'logps/chosen': '-64.55', 'logps/rejected': '-259.6', 'epoch': '0.08889'}

  3%|▎         | 30/1013 [04:12<1:55:47,  7.07s/it]
  3%|▎         | 31/1013 [04:19<1:56:04,  7.09s/it]
  3%|▎         | 32/1013 [04:26<1:54:09,  6.98s/it]
  3%|▎         | 33/1013 [04:33<1:54:16,  7.00s/it]
  3%|▎         | 34/1013 [04:40<1:56:21,  7.13s/it]
  3%|▎         | 35/1013 [04:47<1:55:57,  7.11s/it]
  4%|▎         | 36/1013 [04:54<1:54:24,  7.03s/it]
  4%|▎         | 37/1013 [05:01<1:52:27,  6.91s/it]
  4%|▍         | 38/1013 [05:08<1:55:24,  7.10s/it]
  4%|▍         | 39/1013 [05:15<1:54:45,  7.07s/it]
  4%|▍         | 40/1013 [05:22<1:55:25,  7.12s/it]
                                                   
{'loss': '3.113e-06', 'grad_norm': '0.0004864', 'learning_rate': '1.931e-05', 'entropy': '1.356', 'num_tokens': '3.673e+05', 'logits/chosen': '2.337', 'logits/rejected': '-1.143', 'mean_token_accuracy': '0.667', 'rewards/chosen': '4.367', 'rewards/rejected': '-11.72', 'rewards/accuracies': '1', 'rewards/margins': '16.09', 'logps/chosen': '-61.83', 'logps/rejected': '-295.9', 'epoch': '0.1185'}

  4%|▍         | 40/1013 [05:22<1:55:25,  7.12s/it]
  4%|▍         | 41/1013 [05:30<1:56:39,  7.20s/it]
  4%|▍         | 42/1013 [05:37<1:56:43,  7.21s/it]
  4%|▍         | 43/1013 [05:44<1:53:57,  7.05s/it]
  4%|▍         | 44/1013 [05:51<1:54:20,  7.08s/it]
  4%|▍         | 45/1013 [05:58<1:56:37,  7.23s/it]
  5%|▍         | 46/1013 [06:06<1:58:31,  7.35s/it]
  5%|▍         | 47/1013 [06:13<1:56:24,  7.23s/it]
  5%|▍         | 48/1013 [06:20<1:57:02,  7.28s/it]
  5%|▍         | 49/1013 [06:28<1:58:27,  7.37s/it]
  5%|▍         | 50/1013 [06:35<1:55:07,  7.17s/it]
                                                   
{'loss': '7.578e-06', 'grad_norm': '0.0001202', 'learning_rate': '2.426e-05', 'entropy': '1.32', 'num_tokens': '4.615e+05', 'logits/chosen': '2.315', 'logits/rejected': '-1.057', 'mean_token_accuracy': '0.6602', 'rewards/chosen': '4.112', 'rewards/rejected': '-12.43', 'rewards/accuracies': '1', 'rewards/margins': '16.55', 'logps/chosen': '-63.68', 'logps/rejected': '-302.8', 'epoch': '0.1481'}

  5%|▍         | 50/1013 [06:35<1:55:07,  7.17s/it]
  5%|▌         | 51/1013 [06:42<1:55:56,  7.23s/it]
  5%|▌         | 52/1013 [06:50<1:58:47,  7.42s/it]
  5%|▌         | 53/1013 [06:57<1:55:39,  7.23s/it]
  5%|▌         | 54/1013 [07:04<1:56:24,  7.28s/it]
  5%|▌         | 55/1013 [07:11<1:56:34,  7.30s/it]
  6%|▌         | 56/1013 [07:19<1:55:38,  7.25s/it]
  6%|▌         | 57/1013 [07:26<1:54:14,  7.17s/it]
  6%|▌         | 58/1013 [07:33<1:54:28,  7.19s/it]
  6%|▌         | 59/1013 [07:40<1:54:08,  7.18s/it]
  6%|▌         | 60/1013 [07:47<1:53:30,  7.15s/it]
                                                   
{'loss': '2.057e-06', 'grad_norm': '0.0003872', 'learning_rate': '2.921e-05', 'entropy': '1.327', 'num_tokens': '5.552e+05', 'logits/chosen': '2.262', 'logits/rejected': '-1.086', 'mean_token_accuracy': '0.6665', 'rewards/chosen': '4.521', 'rewards/rejected': '-12.74', 'rewards/accuracies': '1', 'rewards/margins': '17.26', 'logps/chosen': '-63.42', 'logps/rejected': '-304.6', 'epoch': '0.1778'}

  6%|▌         | 60/1013 [07:47<1:53:30,  7.15s/it]
  6%|▌         | 61/1013 [07:54<1:53:42,  7.17s/it]
  6%|▌         | 62/1013 [08:01<1:51:50,  7.06s/it]
  6%|▌         | 63/1013 [08:08<1:52:22,  7.10s/it]
  6%|▋         | 64/1013 [08:15<1:51:52,  7.07s/it]
  6%|▋         | 65/1013 [08:22<1:51:50,  7.08s/it]
  7%|▋         | 66/1013 [08:29<1:50:27,  7.00s/it]
  7%|▋         | 67/1013 [08:36<1:49:35,  6.95s/it]
  7%|▋         | 68/1013 [08:43<1:49:25,  6.95s/it]
  7%|▋         | 69/1013 [08:50<1:50:50,  7.04s/it]
  7%|▋         | 70/1013 [08:58<1:52:31,  7.16s/it]
                                                   
{'loss': '2.663e-05', 'grad_norm': '1.061e-05', 'learning_rate': '3.416e-05', 'entropy': '1.311', 'num_tokens': '6.462e+05', 'logits/chosen': '2.317', 'logits/rejected': '-1.146', 'mean_token_accuracy': '0.6684', 'rewards/chosen': '4.37', 'rewards/rejected': '-12.66', 'rewards/accuracies': '1', 'rewards/margins': '17.03', 'logps/chosen': '-62.94', 'logps/rejected': '-301.5', 'epoch': '0.2074'}

  7%|▋         | 70/1013 [08:58<1:52:31,  7.16s/it]
  7%|▋         | 71/1013 [09:05<1:51:31,  7.10s/it]
  7%|▋         | 72/1013 [09:12<1:51:15,  7.09s/it]
  7%|▋         | 73/1013 [09:19<1:52:22,  7.17s/it]
  7%|▋         | 74/1013 [09:27<1:54:54,  7.34s/it]
  7%|▋         | 75/1013 [09:35<1:57:27,  7.51s/it]
  8%|▊         | 76/1013 [09:42<1:54:29,  7.33s/it]
  8%|▊         | 77/1013 [09:49<1:53:50,  7.30s/it]
  8%|▊         | 78/1013 [09:55<1:49:59,  7.06s/it]
  8%|▊         | 79/1013 [10:02<1:49:07,  7.01s/it]
  8%|▊         | 80/1013 [10:09<1:50:23,  7.10s/it]
                                                   
{'loss': '2.219e-07', 'grad_norm': '8.464e-06', 'learning_rate': '3.911e-05', 'entropy': '1.318', 'num_tokens': '7.414e+05', 'logits/chosen': '2.253', 'logits/rejected': '-1.145', 'mean_token_accuracy': '0.6683', 'rewards/chosen': '4.62', 'rewards/rejected': '-13.44', 'rewards/accuracies': '1', 'rewards/margins': '18.06', 'logps/chosen': '-64.7', 'logps/rejected': '-313.3', 'epoch': '0.237'}

  8%|▊         | 80/1013 [10:09<1:50:23,  7.10s/it]
  8%|▊         | 81/1013 [10:16<1:49:54,  7.08s/it]
  8%|▊         | 82/1013 [10:24<1:52:45,  7.27s/it]
  8%|▊         | 83/1013 [10:31<1:52:43,  7.27s/it]
  8%|▊         | 84/1013 [10:39<1:51:43,  7.22s/it]
  8%|▊         | 85/1013 [10:46<1:52:19,  7.26s/it]
  8%|▊         | 86/1013 [10:53<1:50:38,  7.16s/it]
  9%|▊         | 87/1013 [10:59<1:46:54,  6.93s/it]
  9%|▊         | 88/1013 [11:06<1:47:04,  6.95s/it]
  9%|▉         | 89/1013 [11:14<1:50:18,  7.16s/it]
  9%|▉         | 90/1013 [11:21<1:48:12,  7.03s/it]
                                                   
{'loss': '6.854e-06', 'grad_norm': '0.0001421', 'learning_rate': '4.406e-05', 'entropy': '1.307', 'num_tokens': '8.335e+05', 'logits/chosen': '2.265', 'logits/rejected': '-1.145', 'mean_token_accuracy': '0.6589', 'rewards/chosen': '4.537', 'rewards/rejected': '-13.27', 'rewards/accuracies': '1', 'rewards/margins': '17.81', 'logps/chosen': '-62.47', 'logps/rejected': '-309.7', 'epoch': '0.2667'}

  9%|▉         | 90/1013 [11:21<1:48:12,  7.03s/it]
  9%|▉         | 91/1013 [11:27<1:46:42,  6.94s/it]
  9%|▉         | 92/1013 [11:34<1:47:22,  6.99s/it]
  9%|▉         | 93/1013 [11:42<1:47:54,  7.04s/it]
  9%|▉         | 94/1013 [11:49<1:47:45,  7.04s/it]
  9%|▉         | 95/1013 [11:56<1:49:26,  7.15s/it]
  9%|▉         | 96/1013 [12:03<1:48:32,  7.10s/it]
 10%|▉         | 97/1013 [12:10<1:47:33,  7.05s/it]
 10%|▉         | 98/1013 [12:17<1:48:51,  7.14s/it]
 10%|▉         | 99/1013 [12:24<1:47:02,  7.03s/it]
 10%|▉         | 100/1013 [12:31<1:48:22,  7.12s/it]
                                                    
{'loss': '2e-06', 'grad_norm': '0.0001278', 'learning_rate': '4.901e-05', 'entropy': '1.304', 'num_tokens': '9.253e+05', 'logits/chosen': '2.328', 'logits/rejected': '-1.059', 'mean_token_accuracy': '0.6724', 'rewards/chosen': '4.472', 'rewards/rejected': '-13.28', 'rewards/accuracies': '1', 'rewards/margins': '17.75', 'logps/chosen': '-57.69', 'logps/rejected': '-307.1', 'epoch': '0.2963'}

 10%|▉         | 100/1013 [12:31<1:48:22,  7.12s/it]

  0%|          | 0/143 [00:00<?, ?it/s]

  1%|▏         | 2/143 [00:00<00:22,  6.25it/s]

  2%|▏         | 3/143 [00:00<00:30,  4.64it/s]

  3%|▎         | 4/143 [00:00<00:31,  4.37it/s]

  3%|▎         | 5/143 [00:01<00:34,  3.98it/s]

  4%|▍         | 6/143 [00:01<00:33,  4.09it/s]

  5%|▍         | 7/143 [00:01<00:31,  4.32it/s]

  6%|▌         | 8/143 [00:01<00:33,  4.05it/s]

  6%|▋         | 9/143 [00:02<00:31,  4.30it/s]

  7%|▋         | 10/143 [00:02<00:31,  4.18it/s]

  8%|▊         | 11/143 [00:02<00:31,  4.23it/s]

  8%|▊         | 12/143 [00:02<00:33,  3.94it/s]

  9%|▉         | 13/143 [00:03<00:33,  3.86it/s]

 10%|▉         | 14/143 [00:03<00:35,  3.65it/s]

 10%|█         | 15/143 [00:03<00:34,  3.66it/s]

 11%|█         | 16/143 [00:03<00:31,  3.99it/s]

 12%|█▏        | 17/143 [00:04<00:29,  4.27it/s]

 13%|█▎        | 18/143 [00:04<00:28,  4.35it/s]

 13%|█▎        | 19/143 [00:04<00:32,  3.86it/s]

 14%|█▍        | 20/143 [00:04<00:33,  3.66it/s]

 15%|█▍        | 21/143 [00:05<00:32,  3.74it/s]

 15%|█▌        | 22/143 [00:05<00:32,  3.68it/s]

 16%|█▌        | 23/143 [00:05<00:31,  3.76it/s]

 17%|█▋        | 24/143 [00:06<00:32,  3.71it/s]

 17%|█▋        | 25/143 [00:06<00:30,  3.86it/s]

 18%|█▊        | 26/143 [00:06<00:28,  4.04it/s]

 19%|█▉        | 27/143 [00:06<00:32,  3.59it/s]

 20%|█▉        | 28/143 [00:07<00:33,  3.48it/s]

 20%|██        | 29/143 [00:07<00:32,  3.50it/s]

 21%|██        | 30/143 [00:07<00:32,  3.43it/s]

 22%|██▏       | 31/143 [00:07<00:32,  3.48it/s]

 22%|██▏       | 32/143 [00:08<00:30,  3.67it/s]

 23%|██▎       | 33/143 [00:08<00:28,  3.89it/s]

 24%|██▍       | 34/143 [00:08<00:27,  3.92it/s]

 24%|██▍       | 35/143 [00:08<00:28,  3.81it/s]

 25%|██▌       | 36/143 [00:09<00:26,  4.01it/s]

 26%|██▌       | 37/143 [00:09<00:26,  4.01it/s]

 27%|██▋       | 38/143 [00:09<00:26,  3.99it/s]

 27%|██▋       | 39/143 [00:10<00:27,  3.74it/s]

 28%|██▊       | 40/143 [00:10<00:26,  3.94it/s]

 29%|██▊       | 41/143 [00:10<00:25,  3.95it/s]

 29%|██▉       | 42/143 [00:10<00:27,  3.71it/s]

 30%|███       | 43/143 [00:11<00:27,  3.65it/s]

 31%|███       | 44/143 [00:11<00:26,  3.74it/s]

 31%|███▏      | 45/143 [00:11<00:25,  3.90it/s]

 32%|███▏      | 46/143 [00:12<00:31,  3.13it/s]

 33%|███▎      | 47/143 [00:12<00:29,  3.21it/s]

 34%|███▎      | 48/143 [00:12<00:27,  3.40it/s]

 34%|███▍      | 49/143 [00:12<00:27,  3.45it/s]

 35%|███▍      | 50/143 [00:13<00:26,  3.50it/s]

 36%|███▌      | 51/143 [00:13<00:26,  3.52it/s]

 36%|███▋      | 52/143 [00:13<00:25,  3.63it/s]

 37%|███▋      | 53/143 [00:13<00:23,  3.80it/s]

 38%|███▊      | 54/143 [00:14<00:25,  3.56it/s]

 38%|███▊      | 55/143 [00:14<00:23,  3.67it/s]

 39%|███▉      | 56/143 [00:14<00:22,  3.85it/s]

 40%|███▉      | 57/143 [00:14<00:22,  3.88it/s]

 41%|████      | 58/143 [00:15<00:20,  4.17it/s]

 41%|████▏     | 59/143 [00:15<00:19,  4.28it/s]

 42%|████▏     | 60/143 [00:15<00:20,  4.05it/s]

 43%|████▎     | 61/143 [00:15<00:19,  4.28it/s]

 43%|████▎     | 62/143 [00:16<00:22,  3.64it/s]

 44%|████▍     | 63/143 [00:16<00:21,  3.65it/s]

 45%|████▍     | 64/143 [00:16<00:22,  3.46it/s]

 45%|████▌     | 65/143 [00:17<00:22,  3.45it/s]

 46%|████▌     | 66/143 [00:17<00:24,  3.16it/s]

 47%|████▋     | 67/143 [00:17<00:22,  3.36it/s]

 48%|████▊     | 68/143 [00:18<00:21,  3.45it/s]

 48%|████▊     | 69/143 [00:18<00:23,  3.11it/s]

 49%|████▉     | 70/143 [00:18<00:21,  3.43it/s]

 50%|████▉     | 71/143 [00:18<00:20,  3.47it/s]

 50%|█████     | 72/143 [00:19<00:20,  3.50it/s]

 51%|█████     | 73/143 [00:19<00:20,  3.46it/s]

 52%|█████▏    | 74/143 [00:19<00:20,  3.44it/s]

 52%|█████▏    | 75/143 [00:20<00:20,  3.40it/s]

 53%|█████▎    | 76/143 [00:20<00:18,  3.54it/s]

 54%|█████▍    | 77/143 [00:20<00:17,  3.67it/s]

 55%|█████▍    | 78/143 [00:20<00:16,  3.98it/s]

 55%|█████▌    | 79/143 [00:21<00:16,  3.99it/s]

 56%|█████▌    | 80/143 [00:21<00:18,  3.42it/s]

 57%|█████▋    | 81/143 [00:21<00:16,  3.76it/s]

 57%|█████▋    | 82/143 [00:21<00:17,  3.43it/s]

 58%|█████▊    | 83/143 [00:22<00:18,  3.32it/s]

 59%|█████▊    | 84/143 [00:22<00:17,  3.33it/s]

 59%|█████▉    | 85/143 [00:22<00:16,  3.51it/s]

 60%|██████    | 86/143 [00:23<00:14,  4.05it/s]

 61%|██████    | 87/143 [00:23<00:13,  4.13it/s]

 62%|██████▏   | 88/143 [00:23<00:13,  4.00it/s]

 62%|██████▏   | 89/143 [00:23<00:14,  3.85it/s]

 63%|██████▎   | 90/143 [00:24<00:13,  3.89it/s]

 64%|██████▎   | 91/143 [00:24<00:12,  4.18it/s]

 64%|██████▍   | 92/143 [00:24<00:12,  4.03it/s]

 65%|██████▌   | 93/143 [00:24<00:12,  4.03it/s]

 66%|██████▌   | 94/143 [00:25<00:13,  3.72it/s]

 66%|██████▋   | 95/143 [00:25<00:14,  3.34it/s]

 67%|██████▋   | 96/143 [00:25<00:13,  3.58it/s]

 68%|██████▊   | 97/143 [00:25<00:11,  3.93it/s]

 69%|██████▊   | 98/143 [00:26<00:11,  4.04it/s]

 69%|██████▉   | 99/143 [00:26<00:11,  3.84it/s]

 70%|██████▉   | 100/143 [00:26<00:12,  3.56it/s]

 71%|███████   | 101/143 [00:26<00:11,  3.80it/s]

 71%|███████▏  | 102/143 [00:27<00:10,  3.75it/s]

 72%|███████▏  | 103/143 [00:27<00:11,  3.58it/s]

 73%|███████▎  | 104/143 [00:27<00:10,  3.82it/s]

 73%|███████▎  | 105/143 [00:28<00:10,  3.73it/s]

 74%|███████▍  | 106/143 [00:28<00:10,  3.68it/s]

 75%|███████▍  | 107/143 [00:28<00:09,  3.86it/s]

 76%|███████▌  | 108/143 [00:28<00:08,  3.99it/s]

 76%|███████▌  | 109/143 [00:29<00:08,  4.00it/s]

 77%|███████▋  | 110/143 [00:29<00:08,  4.09it/s]

 78%|███████▊  | 111/143 [00:29<00:08,  3.72it/s]

 78%|███████▊  | 112/143 [00:29<00:07,  3.88it/s]

 79%|███████▉  | 113/143 [00:30<00:08,  3.37it/s]

 80%|███████▉  | 114/143 [00:30<00:08,  3.54it/s]

 80%|████████  | 115/143 [00:30<00:07,  3.54it/s]

 81%|████████  | 116/143 [00:30<00:07,  3.75it/s]

 82%|████████▏ | 117/143 [00:31<00:07,  3.58it/s]

 83%|████████▎ | 118/143 [00:31<00:06,  3.91it/s]

 83%|████████▎ | 119/143 [00:31<00:05,  4.20it/s]

 84%|████████▍ | 120/143 [00:31<00:05,  3.98it/s]

 85%|████████▍ | 121/143 [00:32<00:05,  3.79it/s]

 85%|████████▌ | 122/143 [00:32<00:04,  4.24it/s]

 86%|████████▌ | 123/143 [00:32<00:05,  3.96it/s]

 87%|████████▋ | 124/143 [00:32<00:04,  4.05it/s]

 87%|████████▋ | 125/143 [00:33<00:04,  4.47it/s]

 88%|████████▊ | 126/143 [00:33<00:03,  4.41it/s]

 89%|████████▉ | 127/143 [00:33<00:03,  4.11it/s]

 90%|████████▉ | 128/143 [00:33<00:03,  3.89it/s]

 90%|█████████ | 129/143 [00:34<00:03,  3.99it/s]

 91%|█████████ | 130/143 [00:34<00:03,  4.13it/s]

 92%|█████████▏| 131/143 [00:34<00:02,  4.25it/s]

 92%|█████████▏| 132/143 [00:34<00:02,  4.17it/s]

 93%|█████████▎| 133/143 [00:35<00:02,  3.62it/s]

 94%|█████████▎| 134/143 [00:35<00:02,  3.58it/s]

 94%|█████████▍| 135/143 [00:35<00:02,  3.47it/s]

 95%|█████████▌| 136/143 [00:36<00:01,  3.68it/s]

 96%|█████████▌| 137/143 [00:36<00:01,  3.54it/s]

 97%|█████████▋| 138/143 [00:36<00:01,  3.52it/s]

 97%|█████████▋| 139/143 [00:36<00:01,  3.65it/s]

 98%|█████████▊| 140/143 [00:37<00:00,  3.97it/s]

 99%|█████████▊| 141/143 [00:37<00:00,  3.89it/s]

 99%|█████████▉| 142/143 [00:37<00:00,  3.78it/s]

100%|██████████| 143/143 [00:37<00:00,  3.71it/s]
                                                    

                                                 
{'eval_loss': '3.521e-05', 'eval_runtime': '38.32', 'eval_samples_per_second': '7.437', 'eval_steps_per_second': '3.731', 'eval_entropy': '1.309', 'eval_num_tokens': '9.253e+05', 'eval_logits/chosen': '2.256', 'eval_logits/rejected': '-1.138', 'eval_mean_token_accuracy': '0.6629', 'eval_rewards/chosen': '4.51', 'eval_rewards/rejected': '-13.52', 'eval_rewards/accuracies': '1', 'eval_rewards/margins': '18.03', 'eval_logps/chosen': '-66.05', 'eval_logps/rejected': '-310.3', 'epoch': '0.2963'}

 10%|▉         | 100/1013 [13:10<1:48:22,  7.12s/it]

100%|██████████| 143/143 [00:37<00:00,  3.71it/s]

                                                 

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:38<00:00, 38.01s/it]
Writing model shards: 100%|██████████| 1/1 [00:38<00:00, 38.01s/it]

 10%|▉         | 101/1013 [14:36<10:45:32, 42.47s/it]
 10%|█         | 102/1013 [14:43<8:03:48, 31.86s/it] 
 10%|█         | 103/1013 [14:51<6:10:40, 24.44s/it]
 10%|█         | 104/1013 [14:58<4:51:34, 19.25s/it]
 10%|█         | 105/1013 [15:05<3:56:32, 15.63s/it]
 10%|█         | 106/1013 [15:12<3:17:14, 13.05s/it]
 11%|█         | 107/1013 [15:20<2:53:34, 11.49s/it]
 11%|█         | 108/1013 [15:27<2:35:55, 10.34s/it]
 11%|█         | 109/1013 [15:34<2:20:35,  9.33s/it]
 11%|█         | 110/1013 [15:41<2:08:10,  8.52s/it]
                                                    
{'loss': '3.422e-06', 'grad_norm': '0.0008125', 'learning_rate': '4.999e-05', 'entropy': '1.285', 'num_tokens': '1.016e+06', 'logits/chosen': '2.34', 'logits/rejected': '-1.114', 'mean_token_accuracy': '0.6787', 'rewards/chosen': '4.463', 'rewards/rejected': '-13.3', 'rewards/accuracies': '1', 'rewards/margins': '17.77', 'logps/chosen': '-61.88', 'logps/rejected': '-304.7', 'epoch': '0.3259'}

 11%|█         | 110/1013 [15:41<2:08:10,  8.52s/it]
 11%|█         | 111/1013 [15:49<2:03:52,  8.24s/it]
 11%|█         | 112/1013 [15:55<1:56:17,  7.74s/it]
 11%|█         | 113/1013 [16:02<1:53:28,  7.57s/it]
 11%|█▏        | 114/1013 [16:09<1:50:31,  7.38s/it]
 11%|█▏        | 115/1013 [16:16<1:49:07,  7.29s/it]
 11%|█▏        | 116/1013 [16:23<1:47:25,  7.19s/it]
 12%|█▏        | 117/1013 [16:30<1:46:04,  7.10s/it]
 12%|█▏        | 118/1013 [16:37<1:46:22,  7.13s/it]
 12%|█▏        | 119/1013 [16:44<1:45:18,  7.07s/it]
 12%|█▏        | 120/1013 [16:51<1:44:43,  7.04s/it]
                                                    
{'loss': '1.316e-06', 'grad_norm': '0.000102', 'learning_rate': '4.995e-05', 'entropy': '1.284', 'num_tokens': '1.106e+06', 'logits/chosen': '2.345', 'logits/rejected': '-1.094', 'mean_token_accuracy': '0.671', 'rewards/chosen': '4.635', 'rewards/rejected': '-13.68', 'rewards/accuracies': '1', 'rewards/margins': '18.31', 'logps/chosen': '-61.62', 'logps/rejected': '-307.1', 'epoch': '0.3556'}

 12%|█▏        | 120/1013 [16:51<1:44:43,  7.04s/it]
 12%|█▏        | 121/1013 [16:58<1:44:44,  7.05s/it]
 12%|█▏        | 122/1013 [17:06<1:46:34,  7.18s/it]
 12%|█▏        | 123/1013 [17:13<1:47:04,  7.22s/it]
 12%|█▏        | 124/1013 [17:20<1:43:36,  6.99s/it]
 12%|█▏        | 125/1013 [17:27<1:45:26,  7.12s/it]
 12%|█▏        | 126/1013 [17:34<1:45:34,  7.14s/it]
 13%|█▎        | 127/1013 [17:42<1:47:55,  7.31s/it]
 13%|█▎        | 128/1013 [17:49<1:47:40,  7.30s/it]
 13%|█▎        | 129/1013 [17:57<1:49:32,  7.43s/it]
 13%|█▎        | 130/1013 [18:04<1:49:11,  7.42s/it]
                                                    
{'loss': '2.281e-06', 'grad_norm': '0.0009766', 'learning_rate': '4.988e-05', 'entropy': '1.296', 'num_tokens': '1.198e+06', 'logits/chosen': '2.33', 'logits/rejected': '-1.083', 'mean_token_accuracy': '0.664', 'rewards/chosen': '4.473', 'rewards/rejected': '-13.82', 'rewards/accuracies': '1', 'rewards/margins': '18.29', 'logps/chosen': '-60.91', 'logps/rejected': '-320.1', 'epoch': '0.3852'}

 13%|█▎        | 130/1013 [18:04<1:49:11,  7.42s/it]
 13%|█▎        | 131/1013 [18:11<1:47:26,  7.31s/it]
 13%|█▎        | 132/1013 [18:19<1:46:44,  7.27s/it]
 13%|█▎        | 133/1013 [18:26<1:46:27,  7.26s/it]
 13%|█▎        | 134/1013 [18:33<1:44:36,  7.14s/it]
 13%|█▎        | 135/1013 [18:39<1:42:41,  7.02s/it]
 13%|█▎        | 136/1013 [18:47<1:43:17,  7.07s/it]
 14%|█▎        | 137/1013 [18:53<1:42:11,  7.00s/it]
 14%|█▎        | 138/1013 [19:01<1:45:04,  7.21s/it]
 14%|█▎        | 139/1013 [19:09<1:46:32,  7.31s/it]
 14%|█▍        | 140/1013 [19:16<1:46:17,  7.31s/it]
                                                    
{'loss': '1.216e-06', 'grad_norm': '0.00116', 'learning_rate': '4.979e-05', 'entropy': '1.274', 'num_tokens': '1.289e+06', 'logits/chosen': '2.311', 'logits/rejected': '-1.121', 'mean_token_accuracy': '0.675', 'rewards/chosen': '4.255', 'rewards/rejected': '-13.96', 'rewards/accuracies': '1', 'rewards/margins': '18.21', 'logps/chosen': '-58.56', 'logps/rejected': '-314.1', 'epoch': '0.4148'}

 14%|█▍        | 140/1013 [19:16<1:46:17,  7.31s/it]
 14%|█▍        | 141/1013 [19:23<1:44:41,  7.20s/it]
 14%|█▍        | 142/1013 [19:30<1:44:59,  7.23s/it]
 14%|█▍        | 143/1013 [19:37<1:43:49,  7.16s/it]
 14%|█▍        | 144/1013 [19:45<1:44:02,  7.18s/it]
 14%|█▍        | 145/1013 [19:52<1:43:59,  7.19s/it]
 14%|█▍        | 146/1013 [19:59<1:43:07,  7.14s/it]
 15%|█▍        | 147/1013 [20:06<1:41:35,  7.04s/it]
 15%|█▍        | 148/1013 [20:13<1:43:35,  7.19s/it]
 15%|█▍        | 149/1013 [20:21<1:45:27,  7.32s/it]
 15%|█▍        | 150/1013 [20:28<1:43:39,  7.21s/it]
                                                    
{'loss': '4.051e-07', 'grad_norm': '0.0001659', 'learning_rate': '4.966e-05', 'entropy': '1.285', 'num_tokens': '1.38e+06', 'logits/chosen': '2.333', 'logits/rejected': '-1.181', 'mean_token_accuracy': '0.6699', 'rewards/chosen': '4.356', 'rewards/rejected': '-14.17', 'rewards/accuracies': '1', 'rewards/margins': '18.53', 'logps/chosen': '-60.03', 'logps/rejected': '-315.8', 'epoch': '0.4444'}

 15%|█▍        | 150/1013 [20:28<1:43:39,  7.21s/it]
 15%|█▍        | 151/1013 [20:34<1:41:44,  7.08s/it]
 15%|█▌        | 152/1013 [20:42<1:44:01,  7.25s/it]
 15%|█▌        | 153/1013 [20:49<1:44:08,  7.27s/it]
 15%|█▌        | 154/1013 [20:57<1:44:48,  7.32s/it]
 15%|█▌        | 155/1013 [21:04<1:42:22,  7.16s/it]
 15%|█▌        | 156/1013 [21:11<1:43:08,  7.22s/it]
 15%|█▌        | 157/1013 [21:18<1:44:10,  7.30s/it]
 16%|█▌        | 158/1013 [21:26<1:44:14,  7.31s/it]
 16%|█▌        | 159/1013 [21:33<1:42:34,  7.21s/it]
 16%|█▌        | 160/1013 [21:40<1:42:35,  7.22s/it]
                                                    
{'loss': '2.103e-07', 'grad_norm': '1.013e-05', 'learning_rate': '4.95e-05', 'entropy': '1.301', 'num_tokens': '1.474e+06', 'logits/chosen': '2.325', 'logits/rejected': '-1.09', 'mean_token_accuracy': '0.668', 'rewards/chosen': '4.37', 'rewards/rejected': '-14.26', 'rewards/accuracies': '1', 'rewards/margins': '18.63', 'logps/chosen': '-62.87', 'logps/rejected': '-325.7', 'epoch': '0.4741'}

 16%|█▌        | 160/1013 [21:40<1:42:35,  7.22s/it]
 16%|█▌        | 161/1013 [21:47<1:42:33,  7.22s/it]
 16%|█▌        | 162/1013 [21:55<1:46:32,  7.51s/it]
 16%|█▌        | 163/1013 [22:02<1:43:39,  7.32s/it]
 16%|█▌        | 164/1013 [22:09<1:42:13,  7.22s/it]
 16%|█▋        | 165/1013 [22:17<1:42:37,  7.26s/it]
 16%|█▋        | 166/1013 [22:24<1:41:17,  7.18s/it]
 16%|█▋        | 167/1013 [22:30<1:39:31,  7.06s/it]
 17%|█▋        | 168/1013 [22:38<1:41:37,  7.22s/it]
 17%|█▋        | 169/1013 [22:45<1:41:00,  7.18s/it]
 17%|█▋        | 170/1013 [22:52<1:40:15,  7.14s/it]
                                                    
{'loss': '5.846e-07', 'grad_norm': '0.0001707', 'learning_rate': '4.932e-05', 'entropy': '1.293', 'num_tokens': '1.567e+06', 'logits/chosen': '2.199', 'logits/rejected': '-1.131', 'mean_token_accuracy': '0.6591', 'rewards/chosen': '4.248', 'rewards/rejected': '-13.77', 'rewards/accuracies': '1', 'rewards/margins': '18.02', 'logps/chosen': '-66.82', 'logps/rejected': '-313', 'epoch': '0.5037'}

 17%|█▋        | 170/1013 [22:52<1:40:15,  7.14s/it]
 17%|█▋        | 171/1013 [22:59<1:39:43,  7.11s/it]
 17%|█▋        | 172/1013 [23:06<1:39:07,  7.07s/it]
 17%|█▋        | 173/1013 [23:14<1:40:44,  7.20s/it]
 17%|█▋        | 174/1013 [23:20<1:38:52,  7.07s/it]
 17%|█▋        | 175/1013 [23:28<1:39:51,  7.15s/it]
 17%|█▋        | 176/1013 [23:35<1:38:41,  7.07s/it]
 17%|█▋        | 177/1013 [23:42<1:38:58,  7.10s/it]
 18%|█▊        | 178/1013 [23:50<1:41:14,  7.27s/it]
 18%|█▊        | 179/1013 [23:57<1:40:31,  7.23s/it]
 18%|█▊        | 180/1013 [24:04<1:41:05,  7.28s/it]
                                                    
{'loss': '4.462e-07', 'grad_norm': '1.162e-05', 'learning_rate': '4.91e-05', 'entropy': '1.291', 'num_tokens': '1.659e+06', 'logits/chosen': '2.257', 'logits/rejected': '-1.186', 'mean_token_accuracy': '0.6724', 'rewards/chosen': '4.278', 'rewards/rejected': '-14.01', 'rewards/accuracies': '1', 'rewards/margins': '18.28', 'logps/chosen': '-60.77', 'logps/rejected': '-319.7', 'epoch': '0.5333'}

 18%|█▊        | 180/1013 [24:04<1:41:05,  7.28s/it]
 18%|█▊        | 181/1013 [24:11<1:40:15,  7.23s/it]
 18%|█▊        | 182/1013 [24:18<1:37:47,  7.06s/it]
 18%|█▊        | 183/1013 [24:25<1:39:14,  7.17s/it]
 18%|█▊        | 184/1013 [24:33<1:40:01,  7.24s/it]
 18%|█▊        | 185/1013 [24:39<1:37:14,  7.05s/it]
 18%|█▊        | 186/1013 [24:46<1:37:33,  7.08s/it]
 18%|█▊        | 187/1013 [24:53<1:36:39,  7.02s/it]
 19%|█▊        | 188/1013 [25:00<1:35:15,  6.93s/it]
 19%|█▊        | 189/1013 [25:08<1:39:11,  7.22s/it]
 19%|█▉        | 190/1013 [25:15<1:38:43,  7.20s/it]
                                                    
{'loss': '5.486e-07', 'grad_norm': '6.485e-05', 'learning_rate': '4.886e-05', 'entropy': '1.302', 'num_tokens': '1.75e+06', 'logits/chosen': '2.302', 'logits/rejected': '-1.049', 'mean_token_accuracy': '0.6693', 'rewards/chosen': '4.455', 'rewards/rejected': '-13.99', 'rewards/accuracies': '1', 'rewards/margins': '18.44', 'logps/chosen': '-61.81', 'logps/rejected': '-315.3', 'epoch': '0.563'}

 19%|█▉        | 190/1013 [25:15<1:38:43,  7.20s/it]
 19%|█▉        | 191/1013 [25:22<1:38:35,  7.20s/it]
 19%|█▉        | 192/1013 [25:29<1:38:10,  7.18s/it]
 19%|█▉        | 193/1013 [25:37<1:38:46,  7.23s/it]
 19%|█▉        | 194/1013 [25:44<1:39:48,  7.31s/it]
 19%|█▉        | 195/1013 [25:51<1:36:33,  7.08s/it]
 19%|█▉        | 196/1013 [25:58<1:37:29,  7.16s/it]
 19%|█▉        | 197/1013 [26:05<1:38:00,  7.21s/it]
 20%|█▉        | 198/1013 [26:12<1:36:06,  7.08s/it]
 20%|█▉        | 199/1013 [26:19<1:34:54,  7.00s/it]
 20%|█▉        | 200/1013 [26:26<1:34:20,  6.96s/it]
                                                    
{'loss': '5.882e-06', 'grad_norm': '0.008789', 'learning_rate': '4.859e-05', 'entropy': '1.301', 'num_tokens': '1.841e+06', 'logits/chosen': '2.215', 'logits/rejected': '-1.123', 'mean_token_accuracy': '0.6559', 'rewards/chosen': '4.423', 'rewards/rejected': '-14.15', 'rewards/accuracies': '1', 'rewards/margins': '18.57', 'logps/chosen': '-66.62', 'logps/rejected': '-316.4', 'epoch': '0.5926'}

 20%|█▉        | 200/1013 [26:26<1:34:20,  6.96s/it]

  0%|          | 0/143 [00:00<?, ?it/s]

  1%|▏         | 2/143 [00:00<00:22,  6.19it/s]

  2%|▏         | 3/143 [00:00<00:30,  4.61it/s]

  3%|▎         | 4/143 [00:00<00:31,  4.34it/s]

  3%|▎         | 5/143 [00:01<00:34,  3.97it/s]

  4%|▍         | 6/143 [00:01<00:33,  4.07it/s]

  5%|▍         | 7/143 [00:01<00:31,  4.31it/s]

  6%|▌         | 8/143 [00:01<00:33,  4.05it/s]

  6%|▋         | 9/143 [00:02<00:31,  4.31it/s]

  7%|▋         | 10/143 [00:02<00:31,  4.18it/s]

  8%|▊         | 11/143 [00:02<00:31,  4.23it/s]

  8%|▊         | 12/143 [00:02<00:33,  3.94it/s]

  9%|▉         | 13/143 [00:03<00:33,  3.87it/s]

 10%|▉         | 14/143 [00:03<00:35,  3.65it/s]

 10%|█         | 15/143 [00:03<00:34,  3.67it/s]

 11%|█         | 16/143 [00:03<00:31,  3.97it/s]

 12%|█▏        | 17/143 [00:04<00:29,  4.24it/s]

 13%|█▎        | 18/143 [00:04<00:28,  4.33it/s]

 13%|█▎        | 19/143 [00:04<00:32,  3.86it/s]

 14%|█▍        | 20/143 [00:04<00:33,  3.65it/s]

 15%|█▍        | 21/143 [00:05<00:32,  3.73it/s]

 15%|█▌        | 22/143 [00:05<00:32,  3.69it/s]

 16%|█▌        | 23/143 [00:05<00:31,  3.77it/s]

 17%|█▋        | 24/143 [00:06<00:32,  3.71it/s]

 17%|█▋        | 25/143 [00:06<00:30,  3.87it/s]

 18%|█▊        | 26/143 [00:06<00:28,  4.04it/s]

 19%|█▉        | 27/143 [00:06<00:32,  3.59it/s]

 20%|█▉        | 28/143 [00:07<00:33,  3.48it/s]

 20%|██        | 29/143 [00:07<00:32,  3.49it/s]

 21%|██        | 30/143 [00:07<00:32,  3.43it/s]

 22%|██▏       | 31/143 [00:08<00:32,  3.46it/s]

 22%|██▏       | 32/143 [00:08<00:30,  3.66it/s]

 23%|██▎       | 33/143 [00:08<00:28,  3.89it/s]

 24%|██▍       | 34/143 [00:08<00:27,  3.93it/s]

 24%|██▍       | 35/143 [00:08<00:28,  3.80it/s]

 25%|██▌       | 36/143 [00:09<00:26,  3.99it/s]

 26%|██▌       | 37/143 [00:09<00:26,  3.99it/s]

 27%|██▋       | 38/143 [00:09<00:26,  3.98it/s]

 27%|██▋       | 39/143 [00:10<00:27,  3.73it/s]

 28%|██▊       | 40/143 [00:10<00:26,  3.92it/s]

 29%|██▊       | 41/143 [00:10<00:25,  3.94it/s]

 29%|██▉       | 42/143 [00:10<00:27,  3.71it/s]

 30%|███       | 43/143 [00:11<00:27,  3.64it/s]

 31%|███       | 44/143 [00:11<00:26,  3.73it/s]

 31%|███▏      | 45/143 [00:11<00:25,  3.89it/s]

 32%|███▏      | 46/143 [00:12<00:31,  3.12it/s]

 33%|███▎      | 47/143 [00:12<00:29,  3.20it/s]

 34%|███▎      | 48/143 [00:12<00:27,  3.40it/s]

 34%|███▍      | 49/143 [00:12<00:27,  3.43it/s]

 35%|███▍      | 50/143 [00:13<00:26,  3.48it/s]

 36%|███▌      | 51/143 [00:13<00:26,  3.51it/s]

 36%|███▋      | 52/143 [00:13<00:25,  3.62it/s]

 37%|███▋      | 53/143 [00:13<00:23,  3.80it/s]

 38%|███▊      | 54/143 [00:14<00:25,  3.54it/s]

 38%|███▊      | 55/143 [00:14<00:24,  3.66it/s]

 39%|███▉      | 56/143 [00:14<00:22,  3.84it/s]

 40%|███▉      | 57/143 [00:14<00:22,  3.87it/s]

 41%|████      | 58/143 [00:15<00:20,  4.15it/s]

 41%|████▏     | 59/143 [00:15<00:19,  4.26it/s]

 42%|████▏     | 60/143 [00:15<00:20,  4.03it/s]

 43%|████▎     | 61/143 [00:15<00:19,  4.28it/s]

 43%|████▎     | 62/143 [00:16<00:22,  3.64it/s]

 44%|████▍     | 63/143 [00:16<00:21,  3.66it/s]

 45%|████▍     | 64/143 [00:16<00:22,  3.45it/s]

 45%|████▌     | 65/143 [00:17<00:22,  3.45it/s]

 46%|████▌     | 66/143 [00:17<00:24,  3.16it/s]

 47%|████▋     | 67/143 [00:17<00:22,  3.36it/s]

 48%|████▊     | 68/143 [00:18<00:21,  3.44it/s]

 48%|████▊     | 69/143 [00:18<00:23,  3.11it/s]

 49%|████▉     | 70/143 [00:18<00:21,  3.42it/s]

 50%|████▉     | 71/143 [00:18<00:20,  3.46it/s]

 50%|█████     | 72/143 [00:19<00:20,  3.49it/s]

 51%|█████     | 73/143 [00:19<00:20,  3.45it/s]

 52%|█████▏    | 74/143 [00:19<00:20,  3.44it/s]

 52%|█████▏    | 75/143 [00:20<00:20,  3.39it/s]

 53%|█████▎    | 76/143 [00:20<00:18,  3.53it/s]

 54%|█████▍    | 77/143 [00:20<00:18,  3.65it/s]

 55%|█████▍    | 78/143 [00:20<00:16,  3.96it/s]

 55%|█████▌    | 79/143 [00:21<00:16,  3.98it/s]

 56%|█████▌    | 80/143 [00:21<00:18,  3.41it/s]

 57%|█████▋    | 81/143 [00:21<00:16,  3.75it/s]

 57%|█████▋    | 82/143 [00:22<00:17,  3.40it/s]

 58%|█████▊    | 83/143 [00:22<00:18,  3.29it/s]

 59%|█████▊    | 84/143 [00:22<00:17,  3.31it/s]

 59%|█████▉    | 85/143 [00:22<00:16,  3.48it/s]

 60%|██████    | 86/143 [00:23<00:14,  4.02it/s]

 61%|██████    | 87/143 [00:23<00:13,  4.10it/s]

 62%|██████▏   | 88/143 [00:23<00:13,  3.96it/s]

 62%|██████▏   | 89/143 [00:23<00:14,  3.84it/s]

 63%|██████▎   | 90/143 [00:24<00:13,  3.88it/s]

 64%|██████▎   | 91/143 [00:24<00:12,  4.17it/s]

 64%|██████▍   | 92/143 [00:24<00:12,  4.01it/s]

 65%|██████▌   | 93/143 [00:24<00:12,  4.01it/s]

 66%|██████▌   | 94/143 [00:25<00:13,  3.71it/s]

 66%|██████▋   | 95/143 [00:25<00:14,  3.32it/s]

 67%|██████▋   | 96/143 [00:25<00:13,  3.57it/s]

 68%|██████▊   | 97/143 [00:25<00:11,  3.90it/s]

 69%|██████▊   | 98/143 [00:26<00:11,  4.02it/s]

 69%|██████▉   | 99/143 [00:26<00:11,  3.82it/s]

 70%|██████▉   | 100/143 [00:26<00:12,  3.54it/s]

 71%|███████   | 101/143 [00:27<00:11,  3.79it/s]

 71%|███████▏  | 102/143 [00:27<00:11,  3.72it/s]

 72%|███████▏  | 103/143 [00:27<00:11,  3.55it/s]

 73%|███████▎  | 104/143 [00:27<00:10,  3.80it/s]

 73%|███████▎  | 105/143 [00:28<00:10,  3.73it/s]

 74%|███████▍  | 106/143 [00:28<00:10,  3.68it/s]

 75%|███████▍  | 107/143 [00:28<00:09,  3.84it/s]

 76%|███████▌  | 108/143 [00:28<00:08,  3.98it/s]

 76%|███████▌  | 109/143 [00:29<00:08,  3.98it/s]

 77%|███████▋  | 110/143 [00:29<00:08,  4.09it/s]

 78%|███████▊  | 111/143 [00:29<00:08,  3.70it/s]

 78%|███████▊  | 112/143 [00:29<00:08,  3.85it/s]

 79%|███████▉  | 113/143 [00:30<00:08,  3.36it/s]

 80%|███████▉  | 114/143 [00:30<00:08,  3.53it/s]

 80%|████████  | 115/143 [00:30<00:07,  3.53it/s]

 81%|████████  | 116/143 [00:31<00:07,  3.74it/s]

 82%|████████▏ | 117/143 [00:31<00:07,  3.57it/s]

 83%|████████▎ | 118/143 [00:31<00:06,  3.91it/s]

 83%|████████▎ | 119/143 [00:31<00:05,  4.19it/s]

 84%|████████▍ | 120/143 [00:32<00:05,  3.97it/s]

 85%|████████▍ | 121/143 [00:32<00:05,  3.78it/s]

 85%|████████▌ | 122/143 [00:32<00:04,  4.22it/s]

 86%|████████▌ | 123/143 [00:32<00:05,  3.95it/s]

 87%|████████▋ | 124/143 [00:33<00:04,  4.06it/s]

 87%|████████▋ | 125/143 [00:33<00:04,  4.48it/s]

 88%|████████▊ | 126/143 [00:33<00:03,  4.42it/s]

 89%|████████▉ | 127/143 [00:33<00:03,  4.09it/s]

 90%|████████▉ | 128/143 [00:34<00:03,  3.88it/s]

 90%|█████████ | 129/143 [00:34<00:03,  4.00it/s]

 91%|█████████ | 130/143 [00:34<00:03,  4.12it/s]

 92%|█████████▏| 131/143 [00:34<00:02,  4.23it/s]

 92%|█████████▏| 132/143 [00:34<00:02,  4.15it/s]

 93%|█████████▎| 133/143 [00:35<00:02,  3.62it/s]

 94%|█████████▎| 134/143 [00:35<00:02,  3.59it/s]

 94%|█████████▍| 135/143 [00:35<00:02,  3.46it/s]

 95%|█████████▌| 136/143 [00:36<00:01,  3.67it/s]

 96%|█████████▌| 137/143 [00:36<00:01,  3.53it/s]

 97%|█████████▋| 138/143 [00:36<00:01,  3.51it/s]

 97%|█████████▋| 139/143 [00:36<00:01,  3.63it/s]

 98%|█████████▊| 140/143 [00:37<00:00,  3.96it/s]

 99%|█████████▊| 141/143 [00:37<00:00,  3.88it/s]

 99%|█████████▉| 142/143 [00:37<00:00,  3.77it/s]

100%|██████████| 143/143 [00:37<00:00,  3.83it/s]
                                                    

                                                 
{'eval_loss': '1.247e-05', 'eval_runtime': '38.39', 'eval_samples_per_second': '7.424', 'eval_steps_per_second': '3.725', 'eval_entropy': '1.294', 'eval_num_tokens': '1.841e+06', 'eval_logits/chosen': '2.251', 'eval_logits/rejected': '-1.153', 'eval_mean_token_accuracy': '0.6633', 'eval_rewards/chosen': '4.431', 'eval_rewards/rejected': '-14.31', 'eval_rewards/accuracies': '1', 'eval_rewards/margins': '18.74', 'eval_logps/chosen': '-66.84', 'eval_logps/rejected': '-318.1', 'epoch': '0.5926'}

 20%|█▉        | 200/1013 [27:04<1:34:20,  6.96s/it]

100%|██████████| 143/143 [00:37<00:00,  3.83it/s]

                                                 

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:41<00:00, 41.14s/it]
Writing model shards: 100%|██████████| 1/1 [00:41<00:00, 41.14s/it]

 20%|█▉        | 201/1013 [28:34<9:48:05, 43.46s/it]
 20%|█▉        | 202/1013 [28:41<7:18:26, 32.44s/it]
 20%|██        | 203/1013 [28:48<5:35:32, 24.85s/it]
 20%|██        | 204/1013 [28:55<4:22:40, 19.48s/it]
 20%|██        | 205/1013 [29:03<3:34:25, 15.92s/it]
 20%|██        | 206/1013 [29:10<2:59:46, 13.37s/it]
 20%|██        | 207/1013 [29:18<2:37:01, 11.69s/it]
 21%|██        | 208/1013 [29:26<2:19:53, 10.43s/it]
 21%|██        | 209/1013 [29:32<2:05:31,  9.37s/it]
 21%|██        | 210/1013 [29:39<1:55:43,  8.65s/it]
                                                    
{'loss': '2.41e-07', 'grad_norm': '5.126e-05', 'learning_rate': '4.829e-05', 'entropy': '1.294', 'num_tokens': '1.936e+06', 'logits/chosen': '2.345', 'logits/rejected': '-1.186', 'mean_token_accuracy': '0.6671', 'rewards/chosen': '4.46', 'rewards/rejected': '-14.67', 'rewards/accuracies': '1', 'rewards/margins': '19.13', 'logps/chosen': '-64.12', 'logps/rejected': '-325.7', 'epoch': '0.6222'}

 21%|██        | 210/1013 [29:39<1:55:43,  8.65s/it]
 21%|██        | 211/1013 [29:47<1:52:25,  8.41s/it]
 21%|██        | 212/1013 [29:54<1:46:06,  7.95s/it]
 21%|██        | 213/1013 [30:02<1:44:51,  7.86s/it]
 21%|██        | 214/1013 [30:09<1:40:43,  7.56s/it]
 21%|██        | 215/1013 [30:16<1:38:47,  7.43s/it]
 21%|██▏       | 216/1013 [30:23<1:37:23,  7.33s/it]
 21%|██▏       | 217/1013 [30:31<1:39:10,  7.48s/it]
 22%|██▏       | 218/1013 [30:38<1:36:36,  7.29s/it]
 22%|██▏       | 219/1013 [30:45<1:35:30,  7.22s/it]
 22%|██▏       | 220/1013 [30:52<1:35:25,  7.22s/it]
                                                    
{'loss': '3.415e-07', 'grad_norm': '4.768e-05', 'learning_rate': '4.796e-05', 'entropy': '1.255', 'num_tokens': '2.029e+06', 'logits/chosen': '2.309', 'logits/rejected': '-1.075', 'mean_token_accuracy': '0.6794', 'rewards/chosen': '4.353', 'rewards/rejected': '-14.4', 'rewards/accuracies': '1', 'rewards/margins': '18.75', 'logps/chosen': '-58.98', 'logps/rejected': '-315.6', 'epoch': '0.6519'}

 22%|██▏       | 220/1013 [30:52<1:35:25,  7.22s/it]
 22%|██▏       | 221/1013 [30:59<1:35:57,  7.27s/it]
 22%|██▏       | 222/1013 [31:06<1:34:17,  7.15s/it]
 22%|██▏       | 223/1013 [31:14<1:35:41,  7.27s/it]
 22%|██▏       | 224/1013 [31:21<1:36:20,  7.33s/it]
 22%|██▏       | 225/1013 [31:29<1:36:55,  7.38s/it]
 22%|██▏       | 226/1013 [31:36<1:37:17,  7.42s/it]
 22%|██▏       | 227/1013 [31:43<1:36:31,  7.37s/it]
 23%|██▎       | 228/1013 [31:50<1:35:16,  7.28s/it]
 23%|██▎       | 229/1013 [31:58<1:34:07,  7.20s/it]
 23%|██▎       | 230/1013 [32:04<1:33:01,  7.13s/it]
                                                    
{'loss': '1.383e-06', 'grad_norm': '2.789e-05', 'learning_rate': '4.761e-05', 'entropy': '1.275', 'num_tokens': '2.123e+06', 'logits/chosen': '2.263', 'logits/rejected': '-1.075', 'mean_token_accuracy': '0.666', 'rewards/chosen': '4.317', 'rewards/rejected': '-14.6', 'rewards/accuracies': '1', 'rewards/margins': '18.91', 'logps/chosen': '-64.98', 'logps/rejected': '-320.6', 'epoch': '0.6815'}

 23%|██▎       | 230/1013 [32:04<1:33:01,  7.13s/it]
 23%|██▎       | 231/1013 [32:12<1:33:07,  7.15s/it]
 23%|██▎       | 232/1013 [32:19<1:32:21,  7.10s/it]
 23%|██▎       | 233/1013 [32:26<1:31:35,  7.04s/it]
 23%|██▎       | 234/1013 [32:33<1:31:42,  7.06s/it]
 23%|██▎       | 235/1013 [32:39<1:30:23,  6.97s/it]
 23%|██▎       | 236/1013 [32:46<1:29:54,  6.94s/it]
 23%|██▎       | 237/1013 [32:53<1:30:33,  7.00s/it]
 23%|██▎       | 238/1013 [33:01<1:31:13,  7.06s/it]
 24%|██▎       | 239/1013 [33:08<1:32:23,  7.16s/it]
 24%|██▎       | 240/1013 [33:15<1:32:25,  7.17s/it]
                                                    
{'loss': '1.857e-07', 'grad_norm': '2.789e-05', 'learning_rate': '4.723e-05', 'entropy': '1.289', 'num_tokens': '2.212e+06', 'logits/chosen': '2.338', 'logits/rejected': '-1.152', 'mean_token_accuracy': '0.6677', 'rewards/chosen': '4.302', 'rewards/rejected': '-14.91', 'rewards/accuracies': '1', 'rewards/margins': '19.21', 'logps/chosen': '-62.95', 'logps/rejected': '-325.2', 'epoch': '0.7111'}

 24%|██▎       | 240/1013 [33:15<1:32:25,  7.17s/it]
 24%|██▍       | 241/1013 [33:23<1:33:45,  7.29s/it]
 24%|██▍       | 242/1013 [33:30<1:33:45,  7.30s/it]
 24%|██▍       | 243/1013 [33:38<1:34:22,  7.35s/it]
 24%|██▍       | 244/1013 [33:45<1:36:12,  7.51s/it]
 24%|██▍       | 245/1013 [33:53<1:35:40,  7.48s/it]
 24%|██▍       | 246/1013 [34:00<1:34:26,  7.39s/it]
 24%|██▍       | 247/1013 [34:08<1:35:52,  7.51s/it]
 24%|██▍       | 248/1013 [34:15<1:35:14,  7.47s/it]
 25%|██▍       | 249/1013 [34:23<1:34:28,  7.42s/it]
 25%|██▍       | 250/1013 [34:29<1:32:15,  7.25s/it]
                                                    
{'loss': '4.805e-07', 'grad_norm': '1.365e-05', 'learning_rate': '4.682e-05', 'entropy': '1.298', 'num_tokens': '2.308e+06', 'logits/chosen': '2.357', 'logits/rejected': '-1.215', 'mean_token_accuracy': '0.6871', 'rewards/chosen': '4.415', 'rewards/rejected': '-14.85', 'rewards/accuracies': '1', 'rewards/margins': '19.26', 'logps/chosen': '-61.08', 'logps/rejected': '-327.5', 'epoch': '0.7407'}

 25%|██▍       | 250/1013 [34:29<1:32:15,  7.25s/it]
 25%|██▍       | 251/1013 [34:37<1:31:40,  7.22s/it]
 25%|██▍       | 252/1013 [34:45<1:34:55,  7.48s/it]
 25%|██▍       | 253/1013 [34:53<1:38:19,  7.76s/it]
 25%|██▌       | 254/1013 [35:00<1:36:15,  7.61s/it]
 25%|██▌       | 255/1013 [35:07<1:33:54,  7.43s/it]
 25%|██▌       | 256/1013 [35:15<1:34:48,  7.51s/it]
 25%|██▌       | 257/1013 [35:22<1:32:54,  7.37s/it]
 25%|██▌       | 258/1013 [35:29<1:32:47,  7.37s/it]
 26%|██▌       | 259/1013 [35:37<1:32:20,  7.35s/it]
 26%|██▌       | 260/1013 [35:44<1:30:35,  7.22s/it]
                                                    
{'loss': '1.567e-07', 'grad_norm': '0.0001364', 'learning_rate': '4.639e-05', 'entropy': '1.29', 'num_tokens': '2.404e+06', 'logits/chosen': '2.268', 'logits/rejected': '-1.113', 'mean_token_accuracy': '0.6552', 'rewards/chosen': '4.321', 'rewards/rejected': '-14.58', 'rewards/accuracies': '1', 'rewards/margins': '18.9', 'logps/chosen': '-67.26', 'logps/rejected': '-321.5', 'epoch': '0.7704'}

 26%|██▌       | 260/1013 [35:44<1:30:35,  7.22s/it]
 26%|██▌       | 261/1013 [35:51<1:31:17,  7.28s/it]
 26%|██▌       | 262/1013 [35:58<1:30:22,  7.22s/it]
 26%|██▌       | 263/1013 [36:05<1:28:16,  7.06s/it]
 26%|██▌       | 264/1013 [36:11<1:26:36,  6.94s/it]
 26%|██▌       | 265/1013 [36:19<1:27:26,  7.01s/it]
 26%|██▋       | 266/1013 [36:25<1:26:26,  6.94s/it]
 26%|██▋       | 267/1013 [36:32<1:25:58,  6.91s/it]
 26%|██▋       | 268/1013 [36:40<1:26:58,  7.00s/it]
 27%|██▋       | 269/1013 [36:47<1:26:55,  7.01s/it]
 27%|██▋       | 270/1013 [36:54<1:28:38,  7.16s/it]
                                                    
{'loss': '3.22e-07', 'grad_norm': '2.599e-05', 'learning_rate': '4.593e-05', 'entropy': '1.291', 'num_tokens': '2.494e+06', 'logits/chosen': '2.338', 'logits/rejected': '-1.181', 'mean_token_accuracy': '0.671', 'rewards/chosen': '4.552', 'rewards/rejected': '-14.61', 'rewards/accuracies': '1', 'rewards/margins': '19.16', 'logps/chosen': '-61.25', 'logps/rejected': '-321.8', 'epoch': '0.8'}

 27%|██▋       | 270/1013 [36:54<1:28:38,  7.16s/it]
 27%|██▋       | 271/1013 [37:01<1:28:06,  7.13s/it]
 27%|██▋       | 272/1013 [37:09<1:30:18,  7.31s/it]
 27%|██▋       | 273/1013 [37:16<1:30:48,  7.36s/it]
 27%|██▋       | 274/1013 [37:23<1:28:57,  7.22s/it]
 27%|██▋       | 275/1013 [37:30<1:27:54,  7.15s/it]
 27%|██▋       | 276/1013 [37:37<1:26:49,  7.07s/it]
 27%|██▋       | 277/1013 [37:44<1:27:31,  7.13s/it]
 27%|██▋       | 278/1013 [37:51<1:26:38,  7.07s/it]
 28%|██▊       | 279/1013 [37:58<1:25:30,  6.99s/it]
 28%|██▊       | 280/1013 [38:06<1:27:12,  7.14s/it]
                                                    
{'loss': '1.684e-07', 'grad_norm': '1.073e-05', 'learning_rate': '4.545e-05', 'entropy': '1.297', 'num_tokens': '2.584e+06', 'logits/chosen': '2.262', 'logits/rejected': '-1.199', 'mean_token_accuracy': '0.6727', 'rewards/chosen': '4.606', 'rewards/rejected': '-14.74', 'rewards/accuracies': '1', 'rewards/margins': '19.35', 'logps/chosen': '-62.55', 'logps/rejected': '-328.4', 'epoch': '0.8296'}

 28%|██▊       | 280/1013 [38:06<1:27:12,  7.14s/it]
 28%|██▊       | 281/1013 [38:12<1:25:09,  6.98s/it]
 28%|██▊       | 282/1013 [38:20<1:26:29,  7.10s/it]
 28%|██▊       | 283/1013 [38:27<1:26:22,  7.10s/it]
 28%|██▊       | 284/1013 [38:34<1:26:36,  7.13s/it]
 28%|██▊       | 285/1013 [38:41<1:26:54,  7.16s/it]
 28%|██▊       | 286/1013 [38:48<1:25:21,  7.05s/it]
 28%|██▊       | 287/1013 [38:55<1:26:01,  7.11s/it]
 28%|██▊       | 288/1013 [39:02<1:25:45,  7.10s/it]
 29%|██▊       | 289/1013 [39:09<1:25:45,  7.11s/it]
 29%|██▊       | 290/1013 [39:16<1:24:51,  7.04s/it]
                                                    
{'loss': '1.092e-07', 'grad_norm': '1.472e-05', 'learning_rate': '4.494e-05', 'entropy': '1.271', 'num_tokens': '2.673e+06', 'logits/chosen': '2.304', 'logits/rejected': '-1.16', 'mean_token_accuracy': '0.6695', 'rewards/chosen': '4.521', 'rewards/rejected': '-14.96', 'rewards/accuracies': '1', 'rewards/margins': '19.48', 'logps/chosen': '-64.2', 'logps/rejected': '-321.6', 'epoch': '0.8593'}

 29%|██▊       | 290/1013 [39:16<1:24:51,  7.04s/it]
 29%|██▊       | 291/1013 [39:23<1:24:50,  7.05s/it]
 29%|██▉       | 292/1013 [39:30<1:24:39,  7.04s/it]
 29%|██▉       | 293/1013 [39:38<1:25:47,  7.15s/it]
 29%|██▉       | 294/1013 [39:45<1:25:02,  7.10s/it]
 29%|██▉       | 295/1013 [39:52<1:24:54,  7.10s/it]
 29%|██▉       | 296/1013 [39:59<1:26:42,  7.26s/it]
 29%|██▉       | 297/1013 [40:07<1:28:02,  7.38s/it]
 29%|██▉       | 298/1013 [40:15<1:28:56,  7.46s/it]
 30%|██▉       | 299/1013 [40:21<1:26:19,  7.25s/it]
 30%|██▉       | 300/1013 [40:29<1:26:07,  7.25s/it]
                                                    
{'loss': '9.59e-07', 'grad_norm': '8.047e-06', 'learning_rate': '4.441e-05', 'entropy': '1.259', 'num_tokens': '2.765e+06', 'logits/chosen': '2.245', 'logits/rejected': '-1.205', 'mean_token_accuracy': '0.6734', 'rewards/chosen': '4.453', 'rewards/rejected': '-14.7', 'rewards/accuracies': '1', 'rewards/margins': '19.16', 'logps/chosen': '-63.22', 'logps/rejected': '-316.8', 'epoch': '0.8889'}

 30%|██▉       | 300/1013 [40:29<1:26:07,  7.25s/it]

  0%|          | 0/143 [00:00<?, ?it/s]

  1%|▏         | 2/143 [00:00<00:22,  6.29it/s]

  2%|▏         | 3/143 [00:00<00:30,  4.64it/s]

  3%|▎         | 4/143 [00:00<00:31,  4.36it/s]

  3%|▎         | 5/143 [00:01<00:34,  3.98it/s]

  4%|▍         | 6/143 [00:01<00:33,  4.08it/s]

  5%|▍         | 7/143 [00:01<00:31,  4.31it/s]

  6%|▌         | 8/143 [00:01<00:33,  4.04it/s]

  6%|▋         | 9/143 [00:02<00:31,  4.31it/s]

  7%|▋         | 10/143 [00:02<00:31,  4.18it/s]

  8%|▊         | 11/143 [00:02<00:31,  4.22it/s]

  8%|▊         | 12/143 [00:02<00:33,  3.93it/s]

  9%|▉         | 13/143 [00:03<00:33,  3.86it/s]

 10%|▉         | 14/143 [00:03<00:35,  3.65it/s]

 10%|█         | 15/143 [00:03<00:35,  3.65it/s]

 11%|█         | 16/143 [00:03<00:31,  3.98it/s]

 12%|█▏        | 17/143 [00:04<00:29,  4.26it/s]

 13%|█▎        | 18/143 [00:04<00:28,  4.34it/s]

 13%|█▎        | 19/143 [00:04<00:32,  3.85it/s]

 14%|█▍        | 20/143 [00:04<00:33,  3.65it/s]

 15%|█▍        | 21/143 [00:05<00:32,  3.73it/s]

 15%|█▌        | 22/143 [00:05<00:32,  3.68it/s]

 16%|█▌        | 23/143 [00:05<00:31,  3.75it/s]

 17%|█▋        | 24/143 [00:06<00:32,  3.71it/s]

 17%|█▋        | 25/143 [00:06<00:30,  3.86it/s]

 18%|█▊        | 26/143 [00:06<00:29,  4.02it/s]

 19%|█▉        | 27/143 [00:06<00:32,  3.57it/s]

 20%|█▉        | 28/143 [00:07<00:33,  3.47it/s]

 20%|██        | 29/143 [00:07<00:32,  3.49it/s]

 21%|██        | 30/143 [00:07<00:33,  3.42it/s]

 22%|██▏       | 31/143 [00:08<00:32,  3.46it/s]

 22%|██▏       | 32/143 [00:08<00:30,  3.67it/s]

 23%|██▎       | 33/143 [00:08<00:28,  3.89it/s]

 24%|██▍       | 34/143 [00:08<00:27,  3.92it/s]

 24%|██▍       | 35/143 [00:09<00:28,  3.81it/s]

 25%|██▌       | 36/143 [00:09<00:26,  4.01it/s]

 26%|██▌       | 37/143 [00:09<00:26,  4.02it/s]

 27%|██▋       | 38/143 [00:09<00:26,  3.99it/s]

 27%|██▋       | 39/143 [00:10<00:27,  3.73it/s]

 28%|██▊       | 40/143 [00:10<00:26,  3.93it/s]

 29%|██▊       | 41/143 [00:10<00:25,  3.94it/s]

 29%|██▉       | 42/143 [00:10<00:27,  3.71it/s]

 30%|███       | 43/143 [00:11<00:27,  3.65it/s]

 31%|███       | 44/143 [00:11<00:26,  3.74it/s]

 31%|███▏      | 45/143 [00:11<00:25,  3.89it/s]

 32%|███▏      | 46/143 [00:12<00:31,  3.12it/s]

 33%|███▎      | 47/143 [00:12<00:30,  3.20it/s]

 34%|███▎      | 48/143 [00:12<00:27,  3.40it/s]

 34%|███▍      | 49/143 [00:12<00:27,  3.43it/s]

 35%|███▍      | 50/143 [00:13<00:26,  3.48it/s]

 36%|███▌      | 51/143 [00:13<00:26,  3.52it/s]

 36%|███▋      | 52/143 [00:13<00:25,  3.64it/s]

 37%|███▋      | 53/143 [00:13<00:23,  3.81it/s]

 38%|███▊      | 54/143 [00:14<00:25,  3.56it/s]

 38%|███▊      | 55/143 [00:14<00:23,  3.67it/s]

 39%|███▉      | 56/143 [00:14<00:22,  3.85it/s]

 40%|███▉      | 57/143 [00:14<00:22,  3.89it/s]

 41%|████      | 58/143 [00:15<00:20,  4.15it/s]

 41%|████▏     | 59/143 [00:15<00:19,  4.26it/s]

 42%|████▏     | 60/143 [00:15<00:20,  4.03it/s]

 43%|████▎     | 61/143 [00:15<00:19,  4.29it/s]

 43%|████▎     | 62/143 [00:16<00:22,  3.64it/s]

 44%|████▍     | 63/143 [00:16<00:21,  3.65it/s]

 45%|████▍     | 64/143 [00:16<00:22,  3.46it/s]

 45%|████▌     | 65/143 [00:17<00:22,  3.45it/s]

 46%|████▌     | 66/143 [00:17<00:24,  3.16it/s]

 47%|████▋     | 67/143 [00:17<00:22,  3.35it/s]

 48%|████▊     | 68/143 [00:18<00:21,  3.44it/s]

 48%|████▊     | 69/143 [00:18<00:23,  3.11it/s]

 49%|████▉     | 70/143 [00:18<00:21,  3.42it/s]

 50%|████▉     | 71/143 [00:18<00:20,  3.47it/s]

 50%|█████     | 72/143 [00:19<00:20,  3.49it/s]

 51%|█████     | 73/143 [00:19<00:20,  3.46it/s]

 52%|█████▏    | 74/143 [00:19<00:20,  3.45it/s]

 52%|█████▏    | 75/143 [00:20<00:20,  3.39it/s]

 53%|█████▎    | 76/143 [00:20<00:18,  3.53it/s]

 54%|█████▍    | 77/143 [00:20<00:18,  3.66it/s]

 55%|█████▍    | 78/143 [00:20<00:16,  3.97it/s]

 55%|█████▌    | 79/143 [00:21<00:16,  3.98it/s]

 56%|█████▌    | 80/143 [00:21<00:18,  3.41it/s]

 57%|█████▋    | 81/143 [00:21<00:16,  3.75it/s]

 57%|█████▋    | 82/143 [00:22<00:17,  3.42it/s]

 58%|█████▊    | 83/143 [00:22<00:18,  3.31it/s]

 59%|█████▊    | 84/143 [00:22<00:17,  3.34it/s]

 59%|█████▉    | 85/143 [00:22<00:16,  3.49it/s]

 60%|██████    | 86/143 [00:23<00:14,  4.03it/s]

 61%|██████    | 87/143 [00:23<00:13,  4.12it/s]

 62%|██████▏   | 88/143 [00:23<00:13,  3.98it/s]

 62%|██████▏   | 89/143 [00:23<00:14,  3.84it/s]

 63%|██████▎   | 90/143 [00:24<00:13,  3.87it/s]

 64%|██████▎   | 91/143 [00:24<00:12,  4.17it/s]

 64%|██████▍   | 92/143 [00:24<00:12,  4.01it/s]

 65%|██████▌   | 93/143 [00:24<00:12,  4.00it/s]

 66%|██████▌   | 94/143 [00:25<00:13,  3.72it/s]

 66%|██████▋   | 95/143 [00:25<00:14,  3.33it/s]

 67%|██████▋   | 96/143 [00:25<00:13,  3.56it/s]

 68%|██████▊   | 97/143 [00:25<00:11,  3.91it/s]

 69%|██████▊   | 98/143 [00:26<00:11,  4.02it/s]

 69%|██████▉   | 99/143 [00:26<00:11,  3.81it/s]

 70%|██████▉   | 100/143 [00:26<00:12,  3.55it/s]

 71%|███████   | 101/143 [00:27<00:11,  3.79it/s]

 71%|███████▏  | 102/143 [00:27<00:10,  3.73it/s]

 72%|███████▏  | 103/143 [00:27<00:11,  3.56it/s]

 73%|███████▎  | 104/143 [00:27<00:10,  3.81it/s]

 73%|███████▎  | 105/143 [00:28<00:10,  3.73it/s]

 74%|███████▍  | 106/143 [00:28<00:10,  3.68it/s]

 75%|███████▍  | 107/143 [00:28<00:09,  3.84it/s]

 76%|███████▌  | 108/143 [00:28<00:08,  3.98it/s]

 76%|███████▌  | 109/143 [00:29<00:08,  3.97it/s]

 77%|███████▋  | 110/143 [00:29<00:08,  4.07it/s]

 78%|███████▊  | 111/143 [00:29<00:08,  3.71it/s]

 78%|███████▊  | 112/143 [00:29<00:08,  3.86it/s]

 79%|███████▉  | 113/143 [00:30<00:08,  3.36it/s]

 80%|███████▉  | 114/143 [00:30<00:08,  3.54it/s]

 80%|████████  | 115/143 [00:30<00:07,  3.53it/s]

 81%|████████  | 116/143 [00:31<00:07,  3.73it/s]

 82%|████████▏ | 117/143 [00:31<00:07,  3.57it/s]

 83%|████████▎ | 118/143 [00:31<00:06,  3.91it/s]

 83%|████████▎ | 119/143 [00:31<00:05,  4.20it/s]

 84%|████████▍ | 120/143 [00:32<00:05,  3.98it/s]

 85%|████████▍ | 121/143 [00:32<00:05,  3.79it/s]

 85%|████████▌ | 122/143 [00:32<00:04,  4.23it/s]

 86%|████████▌ | 123/143 [00:32<00:05,  3.95it/s]

 87%|████████▋ | 124/143 [00:33<00:04,  4.06it/s]

 87%|████████▋ | 125/143 [00:33<00:04,  4.49it/s]

 88%|████████▊ | 126/143 [00:33<00:03,  4.43it/s]

 89%|████████▉ | 127/143 [00:33<00:03,  4.10it/s]

 90%|████████▉ | 128/143 [00:33<00:03,  3.87it/s]

 90%|█████████ | 129/143 [00:34<00:03,  4.00it/s]

 91%|█████████ | 130/143 [00:34<00:03,  4.13it/s]

 92%|█████████▏| 131/143 [00:34<00:02,  4.24it/s]

 92%|█████████▏| 132/143 [00:34<00:02,  4.16it/s]

 93%|█████████▎| 133/143 [00:35<00:02,  3.62it/s]

 94%|█████████▎| 134/143 [00:35<00:02,  3.59it/s]

 94%|█████████▍| 135/143 [00:35<00:02,  3.46it/s]

 95%|█████████▌| 136/143 [00:36<00:01,  3.68it/s]

 96%|█████████▌| 137/143 [00:36<00:01,  3.52it/s]

 97%|█████████▋| 138/143 [00:36<00:01,  3.51it/s]

 97%|█████████▋| 139/143 [00:36<00:01,  3.63it/s]

 98%|█████████▊| 140/143 [00:37<00:00,  3.96it/s]

 99%|█████████▊| 141/143 [00:37<00:00,  3.87it/s]

 99%|█████████▉| 142/143 [00:37<00:00,  3.76it/s]

100%|██████████| 143/143 [00:38<00:00,  3.68it/s]
                                                    

                                                 
{'eval_loss': '9.115e-06', 'eval_runtime': '38.41', 'eval_samples_per_second': '7.42', 'eval_steps_per_second': '3.723', 'eval_entropy': '1.286', 'eval_num_tokens': '2.765e+06', 'eval_logits/chosen': '2.245', 'eval_logits/rejected': '-1.158', 'eval_mean_token_accuracy': '0.6628', 'eval_rewards/chosen': '4.364', 'eval_rewards/rejected': '-14.81', 'eval_rewards/accuracies': '1', 'eval_rewards/margins': '19.17', 'eval_logps/chosen': '-67.52', 'eval_logps/rejected': '-323.2', 'epoch': '0.8889'}

 30%|██▉       | 300/1013 [41:07<1:26:07,  7.25s/it]

100%|██████████| 143/143 [00:38<00:00,  3.68it/s]

                                                 

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:41<00:00, 41.39s/it]
Writing model shards: 100%|██████████| 1/1 [00:41<00:00, 41.39s/it]

 30%|██▉       | 301/1013 [42:35<8:29:41, 42.95s/it]
 30%|██▉       | 302/1013 [42:43<6:25:23, 32.52s/it]
 30%|██▉       | 303/1013 [42:52<4:59:06, 25.28s/it]
 30%|███       | 304/1013 [43:00<3:58:18, 20.17s/it]
 30%|███       | 305/1013 [43:08<3:15:13, 16.54s/it]
 30%|███       | 306/1013 [43:16<2:46:04, 14.09s/it]
 30%|███       | 307/1013 [43:25<2:25:15, 12.34s/it]
 30%|███       | 308/1013 [43:33<2:10:10, 11.08s/it]
 31%|███       | 309/1013 [43:40<1:57:36, 10.02s/it]
 31%|███       | 310/1013 [43:49<1:52:15,  9.58s/it]
                                                    
{'loss': '2.076e-07', 'grad_norm': '4.315e-05', 'learning_rate': '4.385e-05', 'entropy': '1.277', 'num_tokens': '2.86e+06', 'logits/chosen': '2.296', 'logits/rejected': '-1.201', 'mean_token_accuracy': '0.6736', 'rewards/chosen': '4.336', 'rewards/rejected': '-15.02', 'rewards/accuracies': '1', 'rewards/margins': '19.35', 'logps/chosen': '-62.47', 'logps/rejected': '-329.2', 'epoch': '0.9185'}

 31%|███       | 310/1013 [43:49<1:52:15,  9.58s/it]
 31%|███       | 311/1013 [43:57<1:47:15,  9.17s/it]
 31%|███       | 312/1013 [44:05<1:42:14,  8.75s/it]
 31%|███       | 313/1013 [44:13<1:39:31,  8.53s/it]
 31%|███       | 314/1013 [44:21<1:37:02,  8.33s/it]
 31%|███       | 315/1013 [44:29<1:36:50,  8.32s/it]
 31%|███       | 316/1013 [44:37<1:36:08,  8.28s/it]
 31%|███▏      | 317/1013 [44:45<1:35:36,  8.24s/it]
 31%|███▏      | 318/1013 [44:53<1:34:46,  8.18s/it]
 31%|███▏      | 319/1013 [45:02<1:34:46,  8.19s/it]
 32%|███▏      | 320/1013 [45:10<1:34:41,  8.20s/it]
                                                    
{'loss': '8.637e-08', 'grad_norm': '2.265e-05', 'learning_rate': '4.328e-05', 'entropy': '1.267', 'num_tokens': '2.951e+06', 'logits/chosen': '2.301', 'logits/rejected': '-1.161', 'mean_token_accuracy': '0.6695', 'rewards/chosen': '4.261', 'rewards/rejected': '-15.03', 'rewards/accuracies': '1', 'rewards/margins': '19.29', 'logps/chosen': '-60.54', 'logps/rejected': '-325.3', 'epoch': '0.9481'}

 32%|███▏      | 320/1013 [45:10<1:34:41,  8.20s/it]
 32%|███▏      | 321/1013 [45:18<1:34:39,  8.21s/it]
 32%|███▏      | 322/1013 [45:26<1:35:21,  8.28s/it]
 32%|███▏      | 323/1013 [45:35<1:35:01,  8.26s/it]
 32%|███▏      | 324/1013 [45:43<1:34:37,  8.24s/it]
 32%|███▏      | 325/1013 [45:51<1:34:05,  8.21s/it]
 32%|███▏      | 326/1013 [45:59<1:34:18,  8.24s/it]
 32%|███▏      | 327/1013 [46:08<1:34:21,  8.25s/it]
 32%|███▏      | 328/1013 [46:16<1:33:12,  8.16s/it]
 32%|███▏      | 329/1013 [46:23<1:31:15,  8.01s/it]
 33%|███▎      | 330/1013 [46:31<1:31:38,  8.05s/it]
                                                    
{'loss': '1.65e-06', 'grad_norm': '3.144e-06', 'learning_rate': '4.268e-05', 'entropy': '1.28', 'num_tokens': '3.046e+06', 'logits/chosen': '2.304', 'logits/rejected': '-1.125', 'mean_token_accuracy': '0.6599', 'rewards/chosen': '4.304', 'rewards/rejected': '-14.87', 'rewards/accuracies': '1', 'rewards/margins': '19.17', 'logps/chosen': '-65.51', 'logps/rejected': '-327.7', 'epoch': '0.9778'}

 33%|███▎      | 330/1013 [46:31<1:31:38,  8.05s/it]
 33%|███▎      | 331/1013 [46:39<1:31:17,  8.03s/it]
 33%|███▎      | 332/1013 [46:48<1:33:09,  8.21s/it]
 33%|███▎      | 333/1013 [46:56<1:32:57,  8.20s/it]
 33%|███▎      | 334/1013 [47:04<1:32:05,  8.14s/it]
 33%|███▎      | 335/1013 [47:12<1:31:20,  8.08s/it]
 33%|███▎      | 336/1013 [47:20<1:31:16,  8.09s/it]
 33%|███▎      | 337/1013 [47:28<1:28:52,  7.89s/it]
 33%|███▎      | 338/1013 [47:32<1:15:35,  6.72s/it]
 33%|███▎      | 339/1013 [47:39<1:18:17,  6.97s/it]
 34%|███▎      | 340/1013 [47:46<1:18:58,  7.04s/it]
                                                    
{'loss': '3.58e-07', 'grad_norm': '5.794e-05', 'learning_rate': '4.206e-05', 'entropy': '1.286', 'num_tokens': '3.131e+06', 'logits/chosen': '2.319', 'logits/rejected': '-1.215', 'mean_token_accuracy': '0.6625', 'rewards/chosen': '4.281', 'rewards/rejected': '-15.31', 'rewards/accuracies': '1', 'rewards/margins': '19.59', 'logps/chosen': '-67.07', 'logps/rejected': '-330.4', 'epoch': '1.006'}

 34%|███▎      | 340/1013 [47:46<1:18:58,  7.04s/it]
 34%|███▎      | 341/1013 [47:54<1:20:43,  7.21s/it]
 34%|███▍      | 342/1013 [48:02<1:22:11,  7.35s/it]
 34%|███▍      | 343/1013 [48:09<1:23:00,  7.43s/it]
 34%|███▍      | 344/1013 [48:17<1:22:46,  7.42s/it]
 34%|███▍      | 345/1013 [48:24<1:22:19,  7.39s/it]
 34%|███▍      | 346/1013 [48:31<1:22:20,  7.41s/it]
 34%|███▍      | 347/1013 [48:39<1:23:31,  7.53s/it]
 34%|███▍      | 348/1013 [48:47<1:25:25,  7.71s/it]
 34%|███▍      | 349/1013 [48:55<1:24:59,  7.68s/it]
 35%|███▍      | 350/1013 [49:03<1:25:20,  7.72s/it]
                                                    
{'loss': '1.532e-07', 'grad_norm': '6.008e-05', 'learning_rate': '4.142e-05', 'entropy': '1.282', 'num_tokens': '3.225e+06', 'logits/chosen': '2.34', 'logits/rejected': '-1.147', 'mean_token_accuracy': '0.6736', 'rewards/chosen': '4.493', 'rewards/rejected': '-15.33', 'rewards/accuracies': '1', 'rewards/margins': '19.83', 'logps/chosen': '-60.32', 'logps/rejected': '-335.2', 'epoch': '1.036'}

 35%|███▍      | 350/1013 [49:03<1:25:20,  7.72s/it]
 35%|███▍      | 351/1013 [49:11<1:25:48,  7.78s/it]
 35%|███▍      | 352/1013 [49:19<1:26:26,  7.85s/it]
 35%|███▍      | 353/1013 [49:27<1:29:24,  8.13s/it]
 35%|███▍      | 354/1013 [49:35<1:26:59,  7.92s/it]
 35%|███▌      | 355/1013 [49:43<1:27:40,  7.99s/it]
 35%|███▌      | 356/1013 [49:52<1:29:16,  8.15s/it]
 35%|███▌      | 357/1013 [50:00<1:29:09,  8.15s/it]
 35%|███▌      | 358/1013 [50:08<1:28:49,  8.14s/it]
 35%|███▌      | 359/1013 [50:16<1:29:01,  8.17s/it]
 36%|███▌      | 360/1013 [50:24<1:29:02,  8.18s/it]
                                                    
{'loss': '4.122e-07', 'grad_norm': '0.0001926', 'learning_rate': '4.076e-05', 'entropy': '1.273', 'num_tokens': '3.319e+06', 'logits/chosen': '2.3', 'logits/rejected': '-1.202', 'mean_token_accuracy': '0.6744', 'rewards/chosen': '4.402', 'rewards/rejected': '-15.23', 'rewards/accuracies': '1', 'rewards/margins': '19.63', 'logps/chosen': '-60.59', 'logps/rejected': '-329.2', 'epoch': '1.065'}

 36%|███▌      | 360/1013 [50:24<1:29:02,  8.18s/it]
 36%|███▌      | 361/1013 [50:32<1:27:12,  8.03s/it]
 36%|███▌      | 362/1013 [50:40<1:26:54,  8.01s/it]
 36%|███▌      | 363/1013 [50:48<1:28:31,  8.17s/it]
 36%|███▌      | 364/1013 [50:57<1:28:39,  8.20s/it]
 36%|███▌      | 365/1013 [51:05<1:27:21,  8.09s/it]
 36%|███▌      | 366/1013 [51:13<1:27:11,  8.09s/it]
 36%|███▌      | 367/1013 [51:20<1:25:46,  7.97s/it]
 36%|███▋      | 368/1013 [51:28<1:24:52,  7.90s/it]
 36%|███▋      | 369/1013 [51:36<1:23:22,  7.77s/it]
 37%|███▋      | 370/1013 [51:43<1:23:12,  7.76s/it]
                                                    
{'loss': '1.151e-06', 'grad_norm': '0.0005913', 'learning_rate': '4.008e-05', 'entropy': '1.277', 'num_tokens': '3.411e+06', 'logits/chosen': '2.288', 'logits/rejected': '-1.155', 'mean_token_accuracy': '0.6719', 'rewards/chosen': '4.197', 'rewards/rejected': '-14.81', 'rewards/accuracies': '1', 'rewards/margins': '19.01', 'logps/chosen': '-63.77', 'logps/rejected': '-319', 'epoch': '1.095'}

 37%|███▋      | 370/1013 [51:43<1:23:12,  7.76s/it]
 37%|███▋      | 371/1013 [51:51<1:23:42,  7.82s/it]
 37%|███▋      | 372/1013 [51:59<1:24:05,  7.87s/it]
 37%|███▋      | 373/1013 [52:08<1:25:21,  8.00s/it]
 37%|███▋      | 374/1013 [52:16<1:25:33,  8.03s/it]
 37%|███▋      | 375/1013 [52:23<1:23:34,  7.86s/it]
 37%|███▋      | 376/1013 [52:31<1:23:39,  7.88s/it]
 37%|███▋      | 377/1013 [52:38<1:21:50,  7.72s/it]
 37%|███▋      | 378/1013 [52:46<1:22:47,  7.82s/it]
 37%|███▋      | 379/1013 [52:54<1:20:47,  7.65s/it]
 38%|███▊      | 380/1013 [53:01<1:20:08,  7.60s/it]
                                                    
{'loss': '1.237e-07', 'grad_norm': '0.0002241', 'learning_rate': '3.939e-05', 'entropy': '1.295', 'num_tokens': '3.504e+06', 'logits/chosen': '2.294', 'logits/rejected': '-1.169', 'mean_token_accuracy': '0.6641', 'rewards/chosen': '4.359', 'rewards/rejected': '-15.51', 'rewards/accuracies': '1', 'rewards/margins': '19.87', 'logps/chosen': '-64.16', 'logps/rejected': '-339.4', 'epoch': '1.124'}

 38%|███▊      | 380/1013 [53:01<1:20:08,  7.60s/it]
 38%|███▊      | 381/1013 [53:09<1:20:27,  7.64s/it]
 38%|███▊      | 382/1013 [53:17<1:22:21,  7.83s/it]
 38%|███▊      | 383/1013 [53:25<1:21:11,  7.73s/it]
 38%|███▊      | 384/1013 [53:32<1:19:35,  7.59s/it]
 38%|███▊      | 385/1013 [53:40<1:20:34,  7.70s/it]
 38%|███▊      | 386/1013 [53:47<1:19:51,  7.64s/it]
 38%|███▊      | 387/1013 [53:55<1:19:57,  7.66s/it]
 38%|███▊      | 388/1013 [54:03<1:20:37,  7.74s/it]
 38%|███▊      | 389/1013 [54:10<1:16:56,  7.40s/it]
 38%|███▊      | 390/1013 [54:17<1:16:25,  7.36s/it]
                                                    
{'loss': '1.195e-07', 'grad_norm': '3.695e-06', 'learning_rate': '3.867e-05', 'entropy': '1.293', 'num_tokens': '3.594e+06', 'logits/chosen': '2.177', 'logits/rejected': '-1.088', 'mean_token_accuracy': '0.657', 'rewards/chosen': '4.462', 'rewards/rejected': '-15.24', 'rewards/accuracies': '1', 'rewards/margins': '19.7', 'logps/chosen': '-69.47', 'logps/rejected': '-327.7', 'epoch': '1.154'}

 38%|███▊      | 390/1013 [54:17<1:16:25,  7.36s/it]
 39%|███▊      | 391/1013 [54:24<1:16:30,  7.38s/it]
 39%|███▊      | 392/1013 [54:32<1:16:02,  7.35s/it]
 39%|███▉      | 393/1013 [54:39<1:14:53,  7.25s/it]
 39%|███▉      | 394/1013 [54:46<1:14:02,  7.18s/it]
 39%|███▉      | 395/1013 [54:53<1:14:00,  7.19s/it]
 39%|███▉      | 396/1013 [55:00<1:14:01,  7.20s/it]
 39%|███▉      | 397/1013 [55:08<1:15:21,  7.34s/it]
 39%|███▉      | 398/1013 [55:15<1:15:46,  7.39s/it]
 39%|███▉      | 399/1013 [55:23<1:15:39,  7.39s/it]
 39%|███▉      | 400/1013 [55:30<1:14:30,  7.29s/it]
                                                    
{'loss': '7.844e-08', 'grad_norm': '8.941e-06', 'learning_rate': '3.794e-05', 'entropy': '1.255', 'num_tokens': '3.683e+06', 'logits/chosen': '2.283', 'logits/rejected': '-1.142', 'mean_token_accuracy': '0.6629', 'rewards/chosen': '4.419', 'rewards/rejected': '-15.36', 'rewards/accuracies': '1', 'rewards/margins': '19.78', 'logps/chosen': '-64.67', 'logps/rejected': '-324.4', 'epoch': '1.184'}

 39%|███▉      | 400/1013 [55:30<1:14:30,  7.29s/it]

  0%|          | 0/143 [00:00<?, ?it/s]

  1%|▏         | 2/143 [00:00<00:22,  6.30it/s]

  2%|▏         | 3/143 [00:00<00:30,  4.64it/s]

  3%|▎         | 4/143 [00:00<00:31,  4.38it/s]

  3%|▎         | 5/143 [00:01<00:34,  3.98it/s]

  4%|▍         | 6/143 [00:01<00:33,  4.08it/s]

  5%|▍         | 7/143 [00:01<00:31,  4.30it/s]

  6%|▌         | 8/143 [00:01<00:33,  4.03it/s]

  6%|▋         | 9/143 [00:02<00:31,  4.29it/s]

  7%|▋         | 10/143 [00:02<00:31,  4.18it/s]

  8%|▊         | 11/143 [00:02<00:31,  4.21it/s]

  8%|▊         | 12/143 [00:02<00:33,  3.92it/s]

  9%|▉         | 13/143 [00:03<00:33,  3.84it/s]

 10%|▉         | 14/143 [00:03<00:35,  3.63it/s]

 10%|█         | 15/143 [00:03<00:35,  3.64it/s]

 11%|█         | 16/143 [00:03<00:32,  3.97it/s]

 12%|█▏        | 17/143 [00:04<00:29,  4.24it/s]

 13%|█▎        | 18/143 [00:04<00:28,  4.32it/s]

 13%|█▎        | 19/143 [00:04<00:32,  3.83it/s]

 14%|█▍        | 20/143 [00:04<00:33,  3.64it/s]

 15%|█▍        | 21/143 [00:05<00:32,  3.72it/s]

 15%|█▌        | 22/143 [00:05<00:33,  3.66it/s]

 16%|█▌        | 23/143 [00:05<00:32,  3.74it/s]

 17%|█▋        | 24/143 [00:06<00:32,  3.70it/s]

 17%|█▋        | 25/143 [00:06<00:30,  3.85it/s]

 18%|█▊        | 26/143 [00:06<00:29,  4.01it/s]

 19%|█▉        | 27/143 [00:06<00:32,  3.56it/s]

 20%|█▉        | 28/143 [00:07<00:33,  3.46it/s]

 20%|██        | 29/143 [00:07<00:32,  3.48it/s]

 21%|██        | 30/143 [00:07<00:33,  3.41it/s]

 22%|██▏       | 31/143 [00:08<00:32,  3.45it/s]

 22%|██▏       | 32/143 [00:08<00:30,  3.65it/s]

 23%|██▎       | 33/143 [00:08<00:28,  3.87it/s]

 24%|██▍       | 34/143 [00:08<00:27,  3.90it/s]

 24%|██▍       | 35/143 [00:09<00:28,  3.79it/s]

 25%|██▌       | 36/143 [00:09<00:26,  3.98it/s]

 26%|██▌       | 37/143 [00:09<00:26,  3.99it/s]

 27%|██▋       | 38/143 [00:09<00:26,  3.97it/s]

 27%|██▋       | 39/143 [00:10<00:27,  3.73it/s]

 28%|██▊       | 40/143 [00:10<00:26,  3.92it/s]

 29%|██▊       | 41/143 [00:10<00:25,  3.94it/s]

 29%|██▉       | 42/143 [00:10<00:27,  3.70it/s]

 30%|███       | 43/143 [00:11<00:27,  3.63it/s]

 31%|███       | 44/143 [00:11<00:26,  3.72it/s]

 31%|███▏      | 45/143 [00:11<00:25,  3.88it/s]

 32%|███▏      | 46/143 [00:12<00:31,  3.12it/s]

 33%|███▎      | 47/143 [00:12<00:30,  3.19it/s]

 34%|███▎      | 48/143 [00:12<00:28,  3.39it/s]

 34%|███▍      | 49/143 [00:12<00:27,  3.42it/s]

 35%|███▍      | 50/143 [00:13<00:26,  3.46it/s]

 36%|███▌      | 51/143 [00:13<00:26,  3.49it/s]

 36%|███▋      | 52/143 [00:13<00:25,  3.61it/s]

 37%|███▋      | 53/143 [00:13<00:23,  3.79it/s]

 38%|███▊      | 54/143 [00:14<00:25,  3.52it/s]

 38%|███▊      | 55/143 [00:14<00:24,  3.64it/s]

 39%|███▉      | 56/143 [00:14<00:22,  3.82it/s]

 40%|███▉      | 57/143 [00:15<00:22,  3.85it/s]

 41%|████      | 58/143 [00:15<00:20,  4.13it/s]

 41%|████▏     | 59/143 [00:15<00:19,  4.23it/s]

 42%|████▏     | 60/143 [00:15<00:20,  4.00it/s]

 43%|████▎     | 61/143 [00:15<00:19,  4.25it/s]

 43%|████▎     | 62/143 [00:16<00:22,  3.61it/s]

 44%|████▍     | 63/143 [00:16<00:22,  3.63it/s]

 45%|████▍     | 64/143 [00:16<00:23,  3.42it/s]

 45%|████▌     | 65/143 [00:17<00:22,  3.42it/s]

 46%|████▌     | 66/143 [00:17<00:24,  3.15it/s]

 47%|████▋     | 67/143 [00:17<00:22,  3.33it/s]

 48%|████▊     | 68/143 [00:18<00:21,  3.43it/s]

 48%|████▊     | 69/143 [00:18<00:23,  3.10it/s]

 49%|████▉     | 70/143 [00:18<00:21,  3.40it/s]

 50%|████▉     | 71/143 [00:19<00:20,  3.46it/s]

 50%|█████     | 72/143 [00:19<00:20,  3.49it/s]

 51%|█████     | 73/143 [00:19<00:20,  3.45it/s]

 52%|█████▏    | 74/143 [00:19<00:20,  3.43it/s]

 52%|█████▏    | 75/143 [00:20<00:20,  3.38it/s]

 53%|█████▎    | 76/143 [00:20<00:19,  3.52it/s]

 54%|█████▍    | 77/143 [00:20<00:18,  3.65it/s]

 55%|█████▍    | 78/143 [00:20<00:16,  3.95it/s]

 55%|█████▌    | 79/143 [00:21<00:16,  3.96it/s]

 56%|█████▌    | 80/143 [00:21<00:18,  3.40it/s]

 57%|█████▋    | 81/143 [00:21<00:16,  3.73it/s]

 57%|█████▋    | 82/143 [00:22<00:17,  3.40it/s]

 58%|█████▊    | 83/143 [00:22<00:18,  3.30it/s]

 59%|█████▊    | 84/143 [00:22<00:17,  3.31it/s]

 59%|█████▉    | 85/143 [00:22<00:16,  3.47it/s]

 60%|██████    | 86/143 [00:23<00:14,  4.00it/s]

 61%|██████    | 87/143 [00:23<00:13,  4.09it/s]

 62%|██████▏   | 88/143 [00:23<00:13,  3.96it/s]

 62%|██████▏   | 89/143 [00:23<00:14,  3.83it/s]

 63%|██████▎   | 90/143 [00:24<00:13,  3.86it/s]

 64%|██████▎   | 91/143 [00:24<00:12,  4.14it/s]

 64%|██████▍   | 92/143 [00:24<00:12,  3.99it/s]

 65%|██████▌   | 93/143 [00:24<00:12,  3.99it/s]

 66%|██████▌   | 94/143 [00:25<00:13,  3.70it/s]

 66%|██████▋   | 95/143 [00:25<00:14,  3.32it/s]

 67%|██████▋   | 96/143 [00:25<00:13,  3.55it/s]

 68%|██████▊   | 97/143 [00:26<00:11,  3.90it/s]

 69%|██████▊   | 98/143 [00:26<00:11,  4.01it/s]

 69%|██████▉   | 99/143 [00:26<00:11,  3.80it/s]

 70%|██████▉   | 100/143 [00:26<00:12,  3.54it/s]

 71%|███████   | 101/143 [00:27<00:11,  3.79it/s]

 71%|███████▏  | 102/143 [00:27<00:10,  3.73it/s]

 72%|███████▏  | 103/143 [00:27<00:11,  3.55it/s]

 73%|███████▎  | 104/143 [00:27<00:10,  3.80it/s]

 73%|███████▎  | 105/143 [00:28<00:10,  3.71it/s]

 74%|███████▍  | 106/143 [00:28<00:10,  3.66it/s]

 75%|███████▍  | 107/143 [00:28<00:09,  3.82it/s]

 76%|███████▌  | 108/143 [00:28<00:08,  3.97it/s]

 76%|███████▌  | 109/143 [00:29<00:08,  3.96it/s]

 77%|███████▋  | 110/143 [00:29<00:08,  4.06it/s]

 78%|███████▊  | 111/143 [00:29<00:08,  3.68it/s]

 78%|███████▊  | 112/143 [00:30<00:08,  3.83it/s]

 79%|███████▉  | 113/143 [00:30<00:08,  3.34it/s]

 80%|███████▉  | 114/143 [00:30<00:08,  3.51it/s]

 80%|████████  | 115/143 [00:30<00:07,  3.52it/s]

 81%|████████  | 116/143 [00:31<00:07,  3.71it/s]

 82%|████████▏ | 117/143 [00:31<00:07,  3.55it/s]

 83%|████████▎ | 118/143 [00:31<00:06,  3.89it/s]

 83%|████████▎ | 119/143 [00:31<00:05,  4.17it/s]

 84%|████████▍ | 120/143 [00:32<00:05,  3.96it/s]

 85%|████████▍ | 121/143 [00:32<00:05,  3.78it/s]

 85%|████████▌ | 122/143 [00:32<00:04,  4.21it/s]

 86%|████████▌ | 123/143 [00:32<00:05,  3.94it/s]

 87%|████████▋ | 124/143 [00:33<00:04,  4.03it/s]

 87%|████████▋ | 125/143 [00:33<00:04,  4.45it/s]

 88%|████████▊ | 126/143 [00:33<00:03,  4.39it/s]

 89%|████████▉ | 127/143 [00:33<00:03,  4.07it/s]

 90%|████████▉ | 128/143 [00:34<00:03,  3.85it/s]

 90%|█████████ | 129/143 [00:34<00:03,  3.98it/s]

 91%|█████████ | 130/143 [00:34<00:03,  4.10it/s]

 92%|█████████▏| 131/143 [00:34<00:02,  4.21it/s]

 92%|█████████▏| 132/143 [00:35<00:02,  4.14it/s]

 93%|█████████▎| 133/143 [00:35<00:02,  3.61it/s]

 94%|█████████▎| 134/143 [00:35<00:02,  3.58it/s]

 94%|█████████▍| 135/143 [00:36<00:02,  3.45it/s]

 95%|█████████▌| 136/143 [00:36<00:01,  3.66it/s]

 96%|█████████▌| 137/143 [00:36<00:01,  3.52it/s]

 97%|█████████▋| 138/143 [00:36<00:01,  3.50it/s]

 97%|█████████▋| 139/143 [00:37<00:01,  3.62it/s]

 98%|█████████▊| 140/143 [00:37<00:00,  3.94it/s]

 99%|█████████▊| 141/143 [00:37<00:00,  3.85it/s]

 99%|█████████▉| 142/143 [00:37<00:00,  3.75it/s]

100%|██████████| 143/143 [00:38<00:00,  3.66it/s]
                                                    

                                                 
{'eval_loss': '7.358e-06', 'eval_runtime': '38.58', 'eval_samples_per_second': '7.387', 'eval_steps_per_second': '3.706', 'eval_entropy': '1.281', 'eval_num_tokens': '3.683e+06', 'eval_logits/chosen': '2.247', 'eval_logits/rejected': '-1.159', 'eval_mean_token_accuracy': '0.6629', 'eval_rewards/chosen': '4.324', 'eval_rewards/rejected': '-15.1', 'eval_rewards/accuracies': '1', 'eval_rewards/margins': '19.42', 'eval_logps/chosen': '-67.91', 'eval_logps/rejected': '-326', 'epoch': '1.184'}

 39%|███▉      | 400/1013 [56:08<1:14:30,  7.29s/it]

100%|██████████| 143/143 [00:38<00:00,  3.66it/s]

                                                 

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:33<00:00, 33.18s/it]
Writing model shards: 100%|██████████| 1/1 [00:33<00:00, 33.18s/it]

 40%|███▉      | 401/1013 [57:27<6:51:04, 40.30s/it]
 40%|███▉      | 402/1013 [57:34<5:09:43, 30.41s/it]
 40%|███▉      | 403/1013 [57:41<3:57:34, 23.37s/it]
 40%|███▉      | 404/1013 [57:49<3:09:06, 18.63s/it]
 40%|███▉      | 405/1013 [57:56<2:35:22, 15.33s/it]
 40%|████      | 406/1013 [58:04<2:09:51, 12.84s/it]
 40%|████      | 407/1013 [58:11<1:52:29, 11.14s/it]
 40%|████      | 408/1013 [58:18<1:42:00, 10.12s/it]
 40%|████      | 409/1013 [58:26<1:33:45,  9.31s/it]
 40%|████      | 410/1013 [58:34<1:29:21,  8.89s/it]
                                                    
{'loss': '1.582e-07', 'grad_norm': '6.05e-06', 'learning_rate': '3.72e-05', 'entropy': '1.279', 'num_tokens': '3.776e+06', 'logits/chosen': '2.235', 'logits/rejected': '-1.204', 'mean_token_accuracy': '0.6723', 'rewards/chosen': '4.321', 'rewards/rejected': '-15.27', 'rewards/accuracies': '1', 'rewards/margins': '19.59', 'logps/chosen': '-61.86', 'logps/rejected': '-331.4', 'epoch': '1.213'}

 40%|████      | 410/1013 [58:34<1:29:21,  8.89s/it]
 41%|████      | 411/1013 [58:41<1:24:26,  8.42s/it]
 41%|████      | 412/1013 [58:48<1:19:57,  7.98s/it]
 41%|████      | 413/1013 [58:56<1:19:37,  7.96s/it]
 41%|████      | 414/1013 [59:03<1:17:34,  7.77s/it]
 41%|████      | 415/1013 [59:11<1:16:10,  7.64s/it]
 41%|████      | 416/1013 [59:18<1:14:15,  7.46s/it]
 41%|████      | 417/1013 [59:25<1:13:48,  7.43s/it]
 41%|████▏     | 418/1013 [59:32<1:13:28,  7.41s/it]
 41%|████▏     | 419/1013 [59:40<1:14:32,  7.53s/it]
 41%|████▏     | 420/1013 [59:48<1:14:50,  7.57s/it]
                                                    
{'loss': '9.945e-08', 'grad_norm': '2.787e-06', 'learning_rate': '3.644e-05', 'entropy': '1.282', 'num_tokens': '3.869e+06', 'logits/chosen': '2.271', 'logits/rejected': '-1.192', 'mean_token_accuracy': '0.6634', 'rewards/chosen': '4.3', 'rewards/rejected': '-15.3', 'rewards/accuracies': '1', 'rewards/margins': '19.6', 'logps/chosen': '-64.48', 'logps/rejected': '-332.1', 'epoch': '1.243'}

 41%|████▏     | 420/1013 [59:48<1:14:50,  7.57s/it]
 42%|████▏     | 421/1013 [59:56<1:14:57,  7.60s/it]
 42%|████▏     | 422/1013 [1:00:03<1:14:09,  7.53s/it]
 42%|████▏     | 423/1013 [1:00:10<1:13:50,  7.51s/it]
 42%|████▏     | 424/1013 [1:00:18<1:12:54,  7.43s/it]
 42%|████▏     | 425/1013 [1:00:25<1:12:37,  7.41s/it]
 42%|████▏     | 426/1013 [1:00:32<1:12:16,  7.39s/it]
 42%|████▏     | 427/1013 [1:00:40<1:14:01,  7.58s/it]
 42%|████▏     | 428/1013 [1:00:48<1:13:27,  7.53s/it]
 42%|████▏     | 429/1013 [1:00:55<1:11:59,  7.40s/it]
 42%|████▏     | 430/1013 [1:01:02<1:12:26,  7.45s/it]
                                                      
{'loss': '2.405e-07', 'grad_norm': '6.294e-05', 'learning_rate': '3.567e-05', 'entropy': '1.276', 'num_tokens': '3.962e+06', 'logits/chosen': '2.305', 'logits/rejected': '-1.136', 'mean_token_accuracy': '0.6751', 'rewards/chosen': '4.183', 'rewards/rejected': '-15.29', 'rewards/accuracies': '1', 'rewards/margins': '19.47', 'logps/chosen': '-64.89', 'logps/rejected': '-334', 'epoch': '1.273'}

 42%|████▏     | 430/1013 [1:01:02<1:12:26,  7.45s/it]
 43%|████▎     | 431/1013 [1:01:10<1:11:33,  7.38s/it]
 43%|████▎     | 432/1013 [1:01:17<1:10:30,  7.28s/it]
 43%|████▎     | 433/1013 [1:01:25<1:12:08,  7.46s/it]
 43%|████▎     | 434/1013 [1:01:32<1:12:10,  7.48s/it]
 43%|████▎     | 435/1013 [1:01:39<1:10:50,  7.35s/it]
 43%|████▎     | 436/1013 [1:01:46<1:10:23,  7.32s/it]
 43%|████▎     | 437/1013 [1:01:54<1:10:39,  7.36s/it]
 43%|████▎     | 438/1013 [1:02:01<1:09:46,  7.28s/it]
 43%|████▎     | 439/1013 [1:02:08<1:10:19,  7.35s/it]
 43%|████▎     | 440/1013 [1:02:16<1:09:22,  7.27s/it]
                                                      
{'loss': '2.914e-07', 'grad_norm': '0.0001268', 'learning_rate': '3.488e-05', 'entropy': '1.28', 'num_tokens': '4.051e+06', 'logits/chosen': '2.346', 'logits/rejected': '-1.232', 'mean_token_accuracy': '0.6791', 'rewards/chosen': '4.32', 'rewards/rejected': '-15.07', 'rewards/accuracies': '1', 'rewards/margins': '19.39', 'logps/chosen': '-58.36', 'logps/rejected': '-320.9', 'epoch': '1.302'}

 43%|████▎     | 440/1013 [1:02:16<1:09:22,  7.27s/it]
 44%|████▎     | 441/1013 [1:02:23<1:10:26,  7.39s/it]
 44%|████▎     | 442/1013 [1:02:31<1:11:32,  7.52s/it]
 44%|████▎     | 443/1013 [1:02:39<1:12:06,  7.59s/it]
 44%|████▍     | 444/1013 [1:02:46<1:10:32,  7.44s/it]
 44%|████▍     | 445/1013 [1:02:53<1:10:41,  7.47s/it]
 44%|████▍     | 446/1013 [1:03:01<1:11:50,  7.60s/it]
 44%|████▍     | 447/1013 [1:03:09<1:11:34,  7.59s/it]
 44%|████▍     | 448/1013 [1:03:16<1:10:30,  7.49s/it]
 44%|████▍     | 449/1013 [1:03:23<1:08:23,  7.28s/it]
 44%|████▍     | 450/1013 [1:03:30<1:09:09,  7.37s/it]
                                                      
{'loss': '1.31e-07', 'grad_norm': '3.934e-05', 'learning_rate': '3.409e-05', 'entropy': '1.261', 'num_tokens': '4.145e+06', 'logits/chosen': '2.359', 'logits/rejected': '-1.181', 'mean_token_accuracy': '0.6829', 'rewards/chosen': '4.355', 'rewards/rejected': '-15.14', 'rewards/accuracies': '1', 'rewards/margins': '19.49', 'logps/chosen': '-57.82', 'logps/rejected': '-327.4', 'epoch': '1.332'}

 44%|████▍     | 450/1013 [1:03:30<1:09:09,  7.37s/it]
 45%|████▍     | 451/1013 [1:03:38<1:08:28,  7.31s/it]
 45%|████▍     | 452/1013 [1:03:45<1:08:56,  7.37s/it]
 45%|████▍     | 453/1013 [1:03:52<1:08:43,  7.36s/it]
 45%|████▍     | 454/1013 [1:04:00<1:07:44,  7.27s/it]
 45%|████▍     | 455/1013 [1:04:07<1:07:55,  7.30s/it]
 45%|████▌     | 456/1013 [1:04:14<1:06:01,  7.11s/it]
 45%|████▌     | 457/1013 [1:04:21<1:06:39,  7.19s/it]
 45%|████▌     | 458/1013 [1:04:28<1:06:49,  7.22s/it]
 45%|████▌     | 459/1013 [1:04:36<1:06:46,  7.23s/it]
 45%|████▌     | 460/1013 [1:04:43<1:08:33,  7.44s/it]
                                                      
{'loss': '1.477e-07', 'grad_norm': '7.153e-06', 'learning_rate': '3.328e-05', 'entropy': '1.272', 'num_tokens': '4.236e+06', 'logits/chosen': '2.284', 'logits/rejected': '-1.066', 'mean_token_accuracy': '0.6687', 'rewards/chosen': '4.341', 'rewards/rejected': '-14.95', 'rewards/accuracies': '1', 'rewards/margins': '19.29', 'logps/chosen': '-65.58', 'logps/rejected': '-324.4', 'epoch': '1.361'}

 45%|████▌     | 460/1013 [1:04:43<1:08:33,  7.44s/it]
 46%|████▌     | 461/1013 [1:04:51<1:08:13,  7.41s/it]
 46%|████▌     | 462/1013 [1:04:59<1:08:53,  7.50s/it]
 46%|████▌     | 463/1013 [1:05:06<1:08:37,  7.49s/it]
 46%|████▌     | 464/1013 [1:05:14<1:09:10,  7.56s/it]
 46%|████▌     | 465/1013 [1:05:21<1:07:53,  7.43s/it]
 46%|████▌     | 466/1013 [1:05:29<1:09:14,  7.59s/it]
 46%|████▌     | 467/1013 [1:05:36<1:07:32,  7.42s/it]
 46%|████▌     | 468/1013 [1:05:43<1:07:53,  7.47s/it]
 46%|████▋     | 469/1013 [1:05:50<1:06:05,  7.29s/it]
 46%|████▋     | 470/1013 [1:05:58<1:06:45,  7.38s/it]
                                                      
{'loss': '2.972e-07', 'grad_norm': '0.0001192', 'learning_rate': '3.246e-05', 'entropy': '1.287', 'num_tokens': '4.328e+06', 'logits/chosen': '2.265', 'logits/rejected': '-1.102', 'mean_token_accuracy': '0.6702', 'rewards/chosen': '4.188', 'rewards/rejected': '-15.2', 'rewards/accuracies': '1', 'rewards/margins': '19.39', 'logps/chosen': '-63.99', 'logps/rejected': '-333', 'epoch': '1.391'}

 46%|████▋     | 470/1013 [1:05:58<1:06:45,  7.38s/it]
 46%|████▋     | 471/1013 [1:06:05<1:06:34,  7.37s/it]
 47%|████▋     | 472/1013 [1:06:13<1:06:16,  7.35s/it]
 47%|████▋     | 473/1013 [1:06:19<1:05:00,  7.22s/it]
 47%|████▋     | 474/1013 [1:06:27<1:05:05,  7.25s/it]
 47%|████▋     | 475/1013 [1:06:34<1:04:39,  7.21s/it]
 47%|████▋     | 476/1013 [1:06:41<1:05:01,  7.27s/it]
 47%|████▋     | 477/1013 [1:06:49<1:07:13,  7.53s/it]
 47%|████▋     | 478/1013 [1:06:57<1:06:35,  7.47s/it]
 47%|████▋     | 479/1013 [1:07:04<1:06:26,  7.47s/it]
 47%|████▋     | 480/1013 [1:07:11<1:05:08,  7.33s/it]
                                                      
{'loss': '2.586e-07', 'grad_norm': '9.871e-05', 'learning_rate': '3.164e-05', 'entropy': '1.294', 'num_tokens': '4.42e+06', 'logits/chosen': '2.307', 'logits/rejected': '-1.166', 'mean_token_accuracy': '0.6737', 'rewards/chosen': '4.177', 'rewards/rejected': '-15.15', 'rewards/accuracies': '1', 'rewards/margins': '19.33', 'logps/chosen': '-61.82', 'logps/rejected': '-332.9', 'epoch': '1.421'}

 47%|████▋     | 480/1013 [1:07:11<1:05:08,  7.33s/it]
 47%|████▋     | 481/1013 [1:07:18<1:03:48,  7.20s/it]
 48%|████▊     | 482/1013 [1:07:26<1:04:17,  7.27s/it]
 48%|████▊     | 483/1013 [1:07:33<1:04:27,  7.30s/it]
 48%|████▊     | 484/1013 [1:07:40<1:04:53,  7.36s/it]
 48%|████▊     | 485/1013 [1:07:47<1:03:22,  7.20s/it]
 48%|████▊     | 486/1013 [1:07:54<1:02:24,  7.11s/it]
 48%|████▊     | 487/1013 [1:08:02<1:05:15,  7.44s/it]
 48%|████▊     | 488/1013 [1:08:10<1:06:08,  7.56s/it]
 48%|████▊     | 489/1013 [1:08:18<1:07:12,  7.70s/it]
 48%|████▊     | 490/1013 [1:08:25<1:05:35,  7.52s/it]
                                                      
{'loss': '8.574e-08', 'grad_norm': '6.771e-05', 'learning_rate': '3.08e-05', 'entropy': '1.276', 'num_tokens': '4.511e+06', 'logits/chosen': '2.339', 'logits/rejected': '-1.179', 'mean_token_accuracy': '0.677', 'rewards/chosen': '4.381', 'rewards/rejected': '-15.2', 'rewards/accuracies': '1', 'rewards/margins': '19.58', 'logps/chosen': '-58.54', 'logps/rejected': '-332.7', 'epoch': '1.45'}

 48%|████▊     | 490/1013 [1:08:25<1:05:35,  7.52s/it]
 48%|████▊     | 491/1013 [1:08:33<1:05:51,  7.57s/it]
 49%|████▊     | 492/1013 [1:08:41<1:06:45,  7.69s/it]
 49%|████▊     | 493/1013 [1:08:49<1:06:43,  7.70s/it]
 49%|████▉     | 494/1013 [1:08:56<1:05:03,  7.52s/it]
 49%|████▉     | 495/1013 [1:09:03<1:04:33,  7.48s/it]
 49%|████▉     | 496/1013 [1:09:11<1:04:47,  7.52s/it]
 49%|████▉     | 497/1013 [1:09:19<1:06:23,  7.72s/it]
 49%|████▉     | 498/1013 [1:09:26<1:04:56,  7.57s/it]
 49%|████▉     | 499/1013 [1:09:33<1:03:49,  7.45s/it]
 49%|████▉     | 500/1013 [1:09:41<1:03:06,  7.38s/it]
                                                      
{'loss': '3.208e-07', 'grad_norm': '8.047e-06', 'learning_rate': '2.996e-05', 'entropy': '1.259', 'num_tokens': '4.605e+06', 'logits/chosen': '2.251', 'logits/rejected': '-1.136', 'mean_token_accuracy': '0.6653', 'rewards/chosen': '4.144', 'rewards/rejected': '-15.39', 'rewards/accuracies': '1', 'rewards/margins': '19.53', 'logps/chosen': '-66', 'logps/rejected': '-332.1', 'epoch': '1.48'}

 49%|████▉     | 500/1013 [1:09:41<1:03:06,  7.38s/it]

  0%|          | 0/143 [00:00<?, ?it/s]

  1%|▏         | 2/143 [00:00<00:22,  6.24it/s]

  2%|▏         | 3/143 [00:00<00:30,  4.62it/s]

  3%|▎         | 4/143 [00:00<00:32,  4.33it/s]

  3%|▎         | 5/143 [00:01<00:34,  3.95it/s]

  4%|▍         | 6/143 [00:01<00:33,  4.06it/s]

  5%|▍         | 7/143 [00:01<00:31,  4.30it/s]

  6%|▌         | 8/143 [00:01<00:33,  4.03it/s]

  6%|▋         | 9/143 [00:02<00:31,  4.29it/s]

  7%|▋         | 10/143 [00:02<00:31,  4.17it/s]

  8%|▊         | 11/143 [00:02<00:31,  4.21it/s]

  8%|▊         | 12/143 [00:02<00:33,  3.92it/s]

  9%|▉         | 13/143 [00:03<00:33,  3.84it/s]

 10%|▉         | 14/143 [00:03<00:35,  3.63it/s]

 10%|█         | 15/143 [00:03<00:35,  3.64it/s]

 11%|█         | 16/143 [00:03<00:32,  3.95it/s]

 12%|█▏        | 17/143 [00:04<00:29,  4.23it/s]

 13%|█▎        | 18/143 [00:04<00:29,  4.31it/s]

 13%|█▎        | 19/143 [00:04<00:32,  3.82it/s]

 14%|█▍        | 20/143 [00:04<00:33,  3.63it/s]

 15%|█▍        | 21/143 [00:05<00:32,  3.71it/s]

 15%|█▌        | 22/143 [00:05<00:33,  3.66it/s]

 16%|█▌        | 23/143 [00:05<00:32,  3.75it/s]

 17%|█▋        | 24/143 [00:06<00:32,  3.69it/s]

 17%|█▋        | 25/143 [00:06<00:30,  3.84it/s]

 18%|█▊        | 26/143 [00:06<00:29,  4.00it/s]

 19%|█▉        | 27/143 [00:06<00:32,  3.56it/s]

 20%|█▉        | 28/143 [00:07<00:33,  3.46it/s]

 20%|██        | 29/143 [00:07<00:32,  3.48it/s]

 21%|██        | 30/143 [00:07<00:33,  3.40it/s]

 22%|██▏       | 31/143 [00:08<00:32,  3.44it/s]

 22%|██▏       | 32/143 [00:08<00:30,  3.66it/s]

 23%|██▎       | 33/143 [00:08<00:28,  3.87it/s]

 24%|██▍       | 34/143 [00:08<00:27,  3.91it/s]

 24%|██▍       | 35/143 [00:09<00:28,  3.79it/s]

 25%|██▌       | 36/143 [00:09<00:26,  3.99it/s]

 26%|██▌       | 37/143 [00:09<00:26,  4.00it/s]

 27%|██▋       | 38/143 [00:09<00:26,  3.98it/s]

 27%|██▋       | 39/143 [00:10<00:28,  3.71it/s]

 28%|██▊       | 40/143 [00:10<00:26,  3.90it/s]

 29%|██▊       | 41/143 [00:10<00:26,  3.91it/s]

 29%|██▉       | 42/143 [00:10<00:27,  3.68it/s]

 30%|███       | 43/143 [00:11<00:27,  3.62it/s]

 31%|███       | 44/143 [00:11<00:26,  3.71it/s]

 31%|███▏      | 45/143 [00:11<00:25,  3.86it/s]

 32%|███▏      | 46/143 [00:12<00:31,  3.10it/s]

 33%|███▎      | 47/143 [00:12<00:30,  3.18it/s]

 34%|███▎      | 48/143 [00:12<00:28,  3.38it/s]

 34%|███▍      | 49/143 [00:12<00:27,  3.41it/s]

 35%|███▍      | 50/143 [00:13<00:26,  3.46it/s]

 36%|███▌      | 51/143 [00:13<00:26,  3.48it/s]

 36%|███▋      | 52/143 [00:13<00:25,  3.60it/s]

 37%|███▋      | 53/143 [00:13<00:23,  3.77it/s]

 38%|███▊      | 54/143 [00:14<00:25,  3.53it/s]

 38%|███▊      | 55/143 [00:14<00:24,  3.64it/s]

 39%|███▉      | 56/143 [00:14<00:22,  3.81it/s]

 40%|███▉      | 57/143 [00:15<00:22,  3.84it/s]

 41%|████      | 58/143 [00:15<00:20,  4.11it/s]

 41%|████▏     | 59/143 [00:15<00:19,  4.23it/s]

 42%|████▏     | 60/143 [00:15<00:20,  4.00it/s]

 43%|████▎     | 61/143 [00:15<00:19,  4.25it/s]

 43%|████▎     | 62/143 [00:16<00:22,  3.61it/s]

 44%|████▍     | 63/143 [00:16<00:22,  3.63it/s]

 45%|████▍     | 64/143 [00:16<00:23,  3.42it/s]

 45%|████▌     | 65/143 [00:17<00:22,  3.42it/s]

 46%|████▌     | 66/143 [00:17<00:24,  3.14it/s]

 47%|████▋     | 67/143 [00:17<00:22,  3.33it/s]

 48%|████▊     | 68/143 [00:18<00:21,  3.43it/s]

 48%|████▊     | 69/143 [00:18<00:23,  3.10it/s]

 49%|████▉     | 70/143 [00:18<00:21,  3.41it/s]

 50%|████▉     | 71/143 [00:19<00:20,  3.46it/s]

 50%|█████     | 72/143 [00:19<00:20,  3.48it/s]

 51%|█████     | 73/143 [00:19<00:20,  3.44it/s]

 52%|█████▏    | 74/143 [00:19<00:20,  3.42it/s]

 52%|█████▏    | 75/143 [00:20<00:20,  3.37it/s]

 53%|█████▎    | 76/143 [00:20<00:19,  3.51it/s]

 54%|█████▍    | 77/143 [00:20<00:18,  3.63it/s]

 55%|█████▍    | 78/143 [00:20<00:16,  3.93it/s]

 55%|█████▌    | 79/143 [00:21<00:16,  3.94it/s]

 56%|█████▌    | 80/143 [00:21<00:18,  3.39it/s]

 57%|█████▋    | 81/143 [00:21<00:16,  3.72it/s]

 57%|█████▋    | 82/143 [00:22<00:17,  3.40it/s]

 58%|█████▊    | 83/143 [00:22<00:18,  3.29it/s]

 59%|█████▊    | 84/143 [00:22<00:17,  3.31it/s]

 59%|█████▉    | 85/143 [00:23<00:16,  3.48it/s]

 60%|██████    | 86/143 [00:23<00:14,  4.00it/s]

 61%|██████    | 87/143 [00:23<00:13,  4.08it/s]

 62%|██████▏   | 88/143 [00:23<00:13,  3.95it/s]

 62%|██████▏   | 89/143 [00:23<00:14,  3.81it/s]

 63%|██████▎   | 90/143 [00:24<00:13,  3.84it/s]

 64%|██████▎   | 91/143 [00:24<00:12,  4.13it/s]

 64%|██████▍   | 92/143 [00:24<00:12,  3.97it/s]

 65%|██████▌   | 93/143 [00:24<00:12,  3.98it/s]

 66%|██████▌   | 94/143 [00:25<00:13,  3.69it/s]

 66%|██████▋   | 95/143 [00:25<00:14,  3.31it/s]

 67%|██████▋   | 96/143 [00:25<00:13,  3.54it/s]

 68%|██████▊   | 97/143 [00:26<00:11,  3.87it/s]

 69%|██████▊   | 98/143 [00:26<00:11,  3.99it/s]

 69%|██████▉   | 99/143 [00:26<00:11,  3.79it/s]

 70%|██████▉   | 100/143 [00:26<00:12,  3.53it/s]

 71%|███████   | 101/143 [00:27<00:11,  3.77it/s]

 71%|███████▏  | 102/143 [00:27<00:11,  3.71it/s]

 72%|███████▏  | 103/143 [00:27<00:11,  3.54it/s]

 73%|███████▎  | 104/143 [00:27<00:10,  3.78it/s]

 73%|███████▎  | 105/143 [00:28<00:10,  3.70it/s]

 74%|███████▍  | 106/143 [00:28<00:10,  3.66it/s]

 75%|███████▍  | 107/143 [00:28<00:09,  3.81it/s]

 76%|███████▌  | 108/143 [00:29<00:08,  3.96it/s]

 76%|███████▌  | 109/143 [00:29<00:08,  3.95it/s]

 77%|███████▋  | 110/143 [00:29<00:08,  4.06it/s]

 78%|███████▊  | 111/143 [00:29<00:08,  3.68it/s]

 78%|███████▊  | 112/143 [00:30<00:08,  3.84it/s]

 79%|███████▉  | 113/143 [00:30<00:08,  3.34it/s]

 80%|███████▉  | 114/143 [00:30<00:08,  3.50it/s]

 80%|████████  | 115/143 [00:31<00:08,  3.49it/s]

 81%|████████  | 116/143 [00:31<00:07,  3.68it/s]

 82%|████████▏ | 117/143 [00:31<00:07,  3.54it/s]

 83%|████████▎ | 118/143 [00:31<00:06,  3.87it/s]

 83%|████████▎ | 119/143 [00:31<00:05,  4.16it/s]

 84%|████████▍ | 120/143 [00:32<00:05,  3.93it/s]

 85%|████████▍ | 121/143 [00:32<00:05,  3.76it/s]

 85%|████████▌ | 122/143 [00:32<00:05,  4.19it/s]

 86%|████████▌ | 123/143 [00:32<00:05,  3.92it/s]

 87%|████████▋ | 124/143 [00:33<00:04,  4.02it/s]

 87%|████████▋ | 125/143 [00:33<00:04,  4.45it/s]

 88%|████████▊ | 126/143 [00:33<00:03,  4.39it/s]

 89%|████████▉ | 127/143 [00:33<00:03,  4.08it/s]

 90%|████████▉ | 128/143 [00:34<00:03,  3.85it/s]

 90%|█████████ | 129/143 [00:34<00:03,  3.96it/s]

 91%|█████████ | 130/143 [00:34<00:03,  4.09it/s]

 92%|█████████▏| 131/143 [00:34<00:02,  4.20it/s]

 92%|█████████▏| 132/143 [00:35<00:02,  4.12it/s]

 93%|█████████▎| 133/143 [00:35<00:02,  3.60it/s]

 94%|█████████▎| 134/143 [00:35<00:02,  3.57it/s]

 94%|█████████▍| 135/143 [00:36<00:02,  3.45it/s]

 95%|█████████▌| 136/143 [00:36<00:01,  3.65it/s]

 96%|█████████▌| 137/143 [00:36<00:01,  3.51it/s]

 97%|█████████▋| 138/143 [00:36<00:01,  3.49it/s]

 97%|█████████▋| 139/143 [00:37<00:01,  3.62it/s]

 98%|█████████▊| 140/143 [00:37<00:00,  3.93it/s]

 99%|█████████▊| 141/143 [00:37<00:00,  3.84it/s]

 99%|█████████▉| 142/143 [00:37<00:00,  3.74it/s]

100%|██████████| 143/143 [00:38<00:00,  3.65it/s]
                                                      

                                                 
{'eval_loss': '7.327e-06', 'eval_runtime': '38.66', 'eval_samples_per_second': '7.371', 'eval_steps_per_second': '3.699', 'eval_entropy': '1.281', 'eval_num_tokens': '4.605e+06', 'eval_logits/chosen': '2.245', 'eval_logits/rejected': '-1.16', 'eval_mean_token_accuracy': '0.6633', 'eval_rewards/chosen': '4.323', 'eval_rewards/rejected': '-15.11', 'eval_rewards/accuracies': '1', 'eval_rewards/margins': '19.44', 'eval_logps/chosen': '-67.92', 'eval_logps/rejected': '-326.2', 'epoch': '1.48'}

 49%|████▉     | 500/1013 [1:10:19<1:03:06,  7.38s/it]

100%|██████████| 143/143 [00:38<00:00,  3.65it/s]

                                                 

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:34<00:00, 34.58s/it]
Writing model shards: 100%|██████████| 1/1 [00:34<00:00, 34.58s/it]

 49%|████▉     | 501/1013 [1:11:43<5:57:21, 41.88s/it]
 50%|████▉     | 502/1013 [1:11:50<4:28:21, 31.51s/it]
 50%|████▉     | 503/1013 [1:11:58<3:26:58, 24.35s/it]
 50%|████▉     | 504/1013 [1:12:05<2:41:52, 19.08s/it]
 50%|████▉     | 505/1013 [1:12:13<2:13:01, 15.71s/it]
 50%|████▉     | 506/1013 [1:12:20<1:51:26, 13.19s/it]
 50%|█████     | 507/1013 [1:12:27<1:34:44, 11.23s/it]
 50%|█████     | 508/1013 [1:12:34<1:25:16, 10.13s/it]
 50%|█████     | 509/1013 [1:12:42<1:18:29,  9.34s/it]
 50%|█████     | 510/1013 [1:12:49<1:13:29,  8.77s/it]
                                                      
{'loss': '1.007e-07', 'grad_norm': '0.0001259', 'learning_rate': '2.911e-05', 'entropy': '1.262', 'num_tokens': '4.695e+06', 'logits/chosen': '2.294', 'logits/rejected': '-1.094', 'mean_token_accuracy': '0.666', 'rewards/chosen': '4.359', 'rewards/rejected': '-15.34', 'rewards/accuracies': '1', 'rewards/margins': '19.7', 'logps/chosen': '-64.26', 'logps/rejected': '-325.9', 'epoch': '1.51'}

 50%|█████     | 510/1013 [1:12:49<1:13:29,  8.77s/it]
 50%|█████     | 511/1013 [1:12:56<1:09:29,  8.31s/it]
 51%|█████     | 512/1013 [1:13:03<1:06:43,  7.99s/it]
 51%|█████     | 513/1013 [1:13:11<1:04:39,  7.76s/it]
 51%|█████     | 514/1013 [1:13:18<1:04:28,  7.75s/it]
 51%|█████     | 515/1013 [1:13:26<1:03:31,  7.65s/it]
 51%|█████     | 516/1013 [1:13:33<1:03:11,  7.63s/it]
 51%|█████     | 517/1013 [1:13:41<1:02:39,  7.58s/it]
 51%|█████     | 518/1013 [1:13:48<1:02:10,  7.54s/it]
 51%|█████     | 519/1013 [1:13:56<1:01:13,  7.44s/it]
 51%|█████▏    | 520/1013 [1:14:03<1:00:03,  7.31s/it]
                                                      
{'loss': '1.422e-07', 'grad_norm': '8.821e-05', 'learning_rate': '2.826e-05', 'entropy': '1.271', 'num_tokens': '4.789e+06', 'logits/chosen': '2.228', 'logits/rejected': '-1.113', 'mean_token_accuracy': '0.6668', 'rewards/chosen': '4.314', 'rewards/rejected': '-14.96', 'rewards/accuracies': '1', 'rewards/margins': '19.27', 'logps/chosen': '-69.59', 'logps/rejected': '-321.3', 'epoch': '1.539'}

 51%|█████▏    | 520/1013 [1:14:03<1:00:03,  7.31s/it]
 51%|█████▏    | 521/1013 [1:14:10<1:00:37,  7.39s/it]
 52%|█████▏    | 522/1013 [1:14:17<1:00:16,  7.37s/it]
 52%|█████▏    | 523/1013 [1:14:26<1:02:22,  7.64s/it]
 52%|█████▏    | 524/1013 [1:14:33<1:00:26,  7.42s/it]
 52%|█████▏    | 525/1013 [1:14:39<58:40,  7.21s/it]  
 52%|█████▏    | 526/1013 [1:14:46<58:13,  7.17s/it]
 52%|█████▏    | 527/1013 [1:14:54<57:53,  7.15s/it]
 52%|█████▏    | 528/1013 [1:15:01<57:37,  7.13s/it]
 52%|█████▏    | 529/1013 [1:15:08<58:48,  7.29s/it]
 52%|█████▏    | 530/1013 [1:15:15<57:39,  7.16s/it]
                                                    
{'loss': '1.384e-07', 'grad_norm': '0.000246', 'learning_rate': '2.741e-05', 'entropy': '1.277', 'num_tokens': '4.878e+06', 'logits/chosen': '2.317', 'logits/rejected': '-1.221', 'mean_token_accuracy': '0.6752', 'rewards/chosen': '4.162', 'rewards/rejected': '-14.95', 'rewards/accuracies': '1', 'rewards/margins': '19.11', 'logps/chosen': '-58.48', 'logps/rejected': '-321.2', 'epoch': '1.569'}

 52%|█████▏    | 530/1013 [1:15:15<57:39,  7.16s/it]
 52%|█████▏    | 531/1013 [1:15:23<58:02,  7.23s/it]
 53%|█████▎    | 532/1013 [1:15:30<57:50,  7.21s/it]
 53%|█████▎    | 533/1013 [1:15:37<56:52,  7.11s/it]
 53%|█████▎    | 534/1013 [1:15:44<57:32,  7.21s/it]
 53%|█████▎    | 535/1013 [1:15:52<59:28,  7.47s/it]
 53%|█████▎    | 536/1013 [1:16:00<1:00:23,  7.60s/it]
 53%|█████▎    | 537/1013 [1:16:07<1:00:03,  7.57s/it]
 53%|█████▎    | 538/1013 [1:16:15<59:18,  7.49s/it]  
 53%|█████▎    | 539/1013 [1:16:22<58:15,  7.38s/it]
 53%|█████▎    | 540/1013 [1:16:29<58:14,  7.39s/it]
                                                    
{'loss': '1.751e-07', 'grad_norm': '3.844e-06', 'learning_rate': '2.655e-05', 'entropy': '1.262', 'num_tokens': '4.97e+06', 'logits/chosen': '2.263', 'logits/rejected': '-1.121', 'mean_token_accuracy': '0.6701', 'rewards/chosen': '4.282', 'rewards/rejected': '-15.32', 'rewards/accuracies': '1', 'rewards/margins': '19.61', 'logps/chosen': '-64.28', 'logps/rejected': '-328.7', 'epoch': '1.599'}

 53%|█████▎    | 540/1013 [1:16:29<58:14,  7.39s/it]
 53%|█████▎    | 541/1013 [1:16:37<59:09,  7.52s/it]
 54%|█████▎    | 542/1013 [1:16:45<59:04,  7.53s/it]
 54%|█████▎    | 543/1013 [1:16:52<59:00,  7.53s/it]
 54%|█████▎    | 544/1013 [1:17:00<58:55,  7.54s/it]
 54%|█████▍    | 545/1013 [1:17:08<59:33,  7.63s/it]
 54%|█████▍    | 546/1013 [1:17:15<58:33,  7.52s/it]
 54%|█████▍    | 547/1013 [1:17:22<58:16,  7.50s/it]
 54%|█████▍    | 548/1013 [1:17:30<58:38,  7.57s/it]
 54%|█████▍    | 549/1013 [1:17:38<58:21,  7.55s/it]
 54%|█████▍    | 550/1013 [1:17:45<57:31,  7.45s/it]
                                                    
{'loss': '1.952e-07', 'grad_norm': '1.49e-05', 'learning_rate': '2.569e-05', 'entropy': '1.266', 'num_tokens': '5.065e+06', 'logits/chosen': '2.304', 'logits/rejected': '-1.12', 'mean_token_accuracy': '0.6727', 'rewards/chosen': '4.242', 'rewards/rejected': '-15.33', 'rewards/accuracies': '1', 'rewards/margins': '19.57', 'logps/chosen': '-63.53', 'logps/rejected': '-332.4', 'epoch': '1.628'}

 54%|█████▍    | 550/1013 [1:17:45<57:31,  7.45s/it]
 54%|█████▍    | 551/1013 [1:17:52<57:47,  7.51s/it]
 54%|█████▍    | 552/1013 [1:18:00<58:09,  7.57s/it]
 55%|█████▍    | 553/1013 [1:18:07<57:14,  7.47s/it]
 55%|█████▍    | 554/1013 [1:18:15<57:06,  7.47s/it]
 55%|█████▍    | 555/1013 [1:18:22<55:36,  7.29s/it]
 55%|█████▍    | 556/1013 [1:18:29<54:59,  7.22s/it]
 55%|█████▍    | 557/1013 [1:18:36<55:04,  7.25s/it]
 55%|█████▌    | 558/1013 [1:18:44<55:41,  7.34s/it]
 55%|█████▌    | 559/1013 [1:18:51<54:49,  7.25s/it]
 55%|█████▌    | 560/1013 [1:18:58<54:38,  7.24s/it]
                                                    
{'loss': '1.471e-07', 'grad_norm': '3.934e-06', 'learning_rate': '2.483e-05', 'entropy': '1.272', 'num_tokens': '5.155e+06', 'logits/chosen': '2.315', 'logits/rejected': '-1.231', 'mean_token_accuracy': '0.6736', 'rewards/chosen': '4.429', 'rewards/rejected': '-15.17', 'rewards/accuracies': '1', 'rewards/margins': '19.6', 'logps/chosen': '-60.61', 'logps/rejected': '-325.5', 'epoch': '1.658'}

 55%|█████▌    | 560/1013 [1:18:58<54:38,  7.24s/it]
 55%|█████▌    | 561/1013 [1:19:05<54:26,  7.23s/it]
 55%|█████▌    | 562/1013 [1:19:13<55:21,  7.36s/it]
 56%|█████▌    | 563/1013 [1:19:20<54:56,  7.32s/it]
 56%|█████▌    | 564/1013 [1:19:28<55:34,  7.43s/it]
 56%|█████▌    | 565/1013 [1:19:35<55:20,  7.41s/it]
 56%|█████▌    | 566/1013 [1:19:43<55:52,  7.50s/it]
 56%|█████▌    | 567/1013 [1:19:50<55:11,  7.42s/it]
 56%|█████▌    | 568/1013 [1:19:57<55:06,  7.43s/it]
 56%|█████▌    | 569/1013 [1:20:05<54:26,  7.36s/it]
 56%|█████▋    | 570/1013 [1:20:12<53:29,  7.25s/it]
                                                    
{'loss': '1.805e-07', 'grad_norm': '0.0003395', 'learning_rate': '2.397e-05', 'entropy': '1.281', 'num_tokens': '5.248e+06', 'logits/chosen': '2.255', 'logits/rejected': '-1.159', 'mean_token_accuracy': '0.6601', 'rewards/chosen': '4.413', 'rewards/rejected': '-15.06', 'rewards/accuracies': '1', 'rewards/margins': '19.47', 'logps/chosen': '-64.34', 'logps/rejected': '-327.6', 'epoch': '1.687'}

 56%|█████▋    | 570/1013 [1:20:12<53:29,  7.25s/it]
 56%|█████▋    | 571/1013 [1:20:19<53:33,  7.27s/it]
 56%|█████▋    | 572/1013 [1:20:26<54:03,  7.36s/it]
 57%|█████▋    | 573/1013 [1:20:34<54:30,  7.43s/it]
 57%|█████▋    | 574/1013 [1:20:42<54:48,  7.49s/it]
 57%|█████▋    | 575/1013 [1:20:49<53:46,  7.37s/it]
 57%|█████▋    | 576/1013 [1:20:56<53:03,  7.29s/it]
 57%|█████▋    | 577/1013 [1:21:04<53:48,  7.40s/it]
 57%|█████▋    | 578/1013 [1:21:11<53:56,  7.44s/it]
 57%|█████▋    | 579/1013 [1:21:19<54:21,  7.52s/it]
 57%|█████▋    | 580/1013 [1:21:26<53:01,  7.35s/it]
                                                    
{'loss': '2.299e-07', 'grad_norm': '0.0001183', 'learning_rate': '2.311e-05', 'entropy': '1.269', 'num_tokens': '5.341e+06', 'logits/chosen': '2.278', 'logits/rejected': '-1.179', 'mean_token_accuracy': '0.667', 'rewards/chosen': '4.299', 'rewards/rejected': '-15.06', 'rewards/accuracies': '1', 'rewards/margins': '19.36', 'logps/chosen': '-66.66', 'logps/rejected': '-323.9', 'epoch': '1.717'}

 57%|█████▋    | 580/1013 [1:21:26<53:01,  7.35s/it]
 57%|█████▋    | 581/1013 [1:21:33<52:57,  7.36s/it]
 57%|█████▋    | 582/1013 [1:21:40<52:03,  7.25s/it]
 58%|█████▊    | 583/1013 [1:21:48<53:24,  7.45s/it]
 58%|█████▊    | 584/1013 [1:21:56<54:23,  7.61s/it]
 58%|█████▊    | 585/1013 [1:22:03<53:40,  7.52s/it]
 58%|█████▊    | 586/1013 [1:22:11<52:51,  7.43s/it]
 58%|█████▊    | 587/1013 [1:22:18<52:27,  7.39s/it]
 58%|█████▊    | 588/1013 [1:22:25<52:29,  7.41s/it]
 58%|█████▊    | 589/1013 [1:22:33<52:15,  7.40s/it]
 58%|█████▊    | 590/1013 [1:22:40<51:33,  7.31s/it]
                                                    
{'loss': '5.122e-07', 'grad_norm': '9.894e-06', 'learning_rate': '2.225e-05', 'entropy': '1.303', 'num_tokens': '5.433e+06', 'logits/chosen': '2.242', 'logits/rejected': '-1.124', 'mean_token_accuracy': '0.658', 'rewards/chosen': '4.279', 'rewards/rejected': '-14.9', 'rewards/accuracies': '1', 'rewards/margins': '19.18', 'logps/chosen': '-66.86', 'logps/rejected': '-326.8', 'epoch': '1.747'}

 58%|█████▊    | 590/1013 [1:22:40<51:33,  7.31s/it]
 58%|█████▊    | 591/1013 [1:22:47<51:09,  7.27s/it]
 58%|█████▊    | 592/1013 [1:22:55<52:36,  7.50s/it]
 59%|█████▊    | 593/1013 [1:23:02<51:59,  7.43s/it]
 59%|█████▊    | 594/1013 [1:23:10<51:59,  7.45s/it]
 59%|█████▊    | 595/1013 [1:23:17<52:26,  7.53s/it]
 59%|█████▉    | 596/1013 [1:23:25<52:53,  7.61s/it]
 59%|█████▉    | 597/1013 [1:23:33<52:30,  7.57s/it]
 59%|█████▉    | 598/1013 [1:23:40<52:26,  7.58s/it]
 59%|█████▉    | 599/1013 [1:23:48<52:11,  7.56s/it]
 59%|█████▉    | 600/1013 [1:23:55<51:42,  7.51s/it]
                                                    
{'loss': '5.188e-07', 'grad_norm': '1.812e-05', 'learning_rate': '2.14e-05', 'entropy': '1.251', 'num_tokens': '5.528e+06', 'logits/chosen': '2.429', 'logits/rejected': '-1.098', 'mean_token_accuracy': '0.6845', 'rewards/chosen': '4.607', 'rewards/rejected': '-15.17', 'rewards/accuracies': '1', 'rewards/margins': '19.78', 'logps/chosen': '-58.96', 'logps/rejected': '-328.3', 'epoch': '1.776'}

 59%|█████▉    | 600/1013 [1:23:55<51:42,  7.51s/it]

  0%|          | 0/143 [00:00<?, ?it/s]

  1%|▏         | 2/143 [00:00<00:22,  6.30it/s]

  2%|▏         | 3/143 [00:00<00:30,  4.63it/s]

  3%|▎         | 4/143 [00:00<00:32,  4.34it/s]

  3%|▎         | 5/143 [00:01<00:34,  3.97it/s]

  4%|▍         | 6/143 [00:01<00:33,  4.04it/s]

  5%|▍         | 7/143 [00:01<00:31,  4.27it/s]

  6%|▌         | 8/143 [00:01<00:33,  4.01it/s]

  6%|▋         | 9/143 [00:02<00:31,  4.26it/s]

  7%|▋         | 10/143 [00:02<00:32,  4.15it/s]

  8%|▊         | 11/143 [00:02<00:31,  4.19it/s]

  8%|▊         | 12/143 [00:02<00:33,  3.91it/s]

  9%|▉         | 13/143 [00:03<00:34,  3.82it/s]

 10%|▉         | 14/143 [00:03<00:35,  3.62it/s]

 10%|█         | 15/143 [00:03<00:35,  3.63it/s]

 11%|█         | 16/143 [00:03<00:32,  3.95it/s]

 12%|█▏        | 17/143 [00:04<00:29,  4.22it/s]

 13%|█▎        | 18/143 [00:04<00:29,  4.30it/s]

 13%|█▎        | 19/143 [00:04<00:32,  3.83it/s]

 14%|█▍        | 20/143 [00:04<00:33,  3.63it/s]

 15%|█▍        | 21/143 [00:05<00:32,  3.71it/s]

 15%|█▌        | 22/143 [00:05<00:33,  3.67it/s]

 16%|█▌        | 23/143 [00:05<00:32,  3.74it/s]

 17%|█▋        | 24/143 [00:06<00:32,  3.67it/s]

 17%|█▋        | 25/143 [00:06<00:30,  3.82it/s]

 18%|█▊        | 26/143 [00:06<00:29,  3.98it/s]

 19%|█▉        | 27/143 [00:06<00:32,  3.55it/s]

 20%|█▉        | 28/143 [00:07<00:33,  3.44it/s]

 20%|██        | 29/143 [00:07<00:32,  3.46it/s]

 21%|██        | 30/143 [00:07<00:33,  3.39it/s]

 22%|██▏       | 31/143 [00:08<00:32,  3.43it/s]

 22%|██▏       | 32/143 [00:08<00:30,  3.64it/s]

 23%|██▎       | 33/143 [00:08<00:28,  3.85it/s]

 24%|██▍       | 34/143 [00:08<00:28,  3.88it/s]

 24%|██▍       | 35/143 [00:09<00:28,  3.77it/s]

 25%|██▌       | 36/143 [00:09<00:27,  3.96it/s]

 26%|██▌       | 37/143 [00:09<00:26,  3.96it/s]

 27%|██▋       | 38/143 [00:09<00:26,  3.95it/s]

 27%|██▋       | 39/143 [00:10<00:28,  3.71it/s]

 28%|██▊       | 40/143 [00:10<00:26,  3.89it/s]

 29%|██▊       | 41/143 [00:10<00:26,  3.90it/s]

 29%|██▉       | 42/143 [00:10<00:27,  3.67it/s]

 30%|███       | 43/143 [00:11<00:27,  3.62it/s]

 31%|███       | 44/143 [00:11<00:26,  3.70it/s]

 31%|███▏      | 45/143 [00:11<00:25,  3.86it/s]

 32%|███▏      | 46/143 [00:12<00:31,  3.10it/s]

 33%|███▎      | 47/143 [00:12<00:30,  3.17it/s]

 34%|███▎      | 48/143 [00:12<00:28,  3.37it/s]

 34%|███▍      | 49/143 [00:12<00:27,  3.41it/s]

 35%|███▍      | 50/143 [00:13<00:26,  3.46it/s]

 36%|███▌      | 51/143 [00:13<00:26,  3.49it/s]

 36%|███▋      | 52/143 [00:13<00:25,  3.60it/s]

 37%|███▋      | 53/143 [00:14<00:23,  3.76it/s]

 38%|███▊      | 54/143 [00:14<00:25,  3.51it/s]

 38%|███▊      | 55/143 [00:14<00:24,  3.61it/s]

 39%|███▉      | 56/143 [00:14<00:23,  3.78it/s]

 40%|███▉      | 57/143 [00:15<00:22,  3.81it/s]

 41%|████      | 58/143 [00:15<00:20,  4.09it/s]

 41%|████▏     | 59/143 [00:15<00:19,  4.20it/s]

 42%|████▏     | 60/143 [00:15<00:20,  3.99it/s]

 43%|████▎     | 61/143 [00:16<00:19,  4.24it/s]

 43%|████▎     | 62/143 [00:16<00:22,  3.61it/s]

 44%|████▍     | 63/143 [00:16<00:22,  3.63it/s]

 45%|████▍     | 64/143 [00:16<00:23,  3.43it/s]

 45%|████▌     | 65/143 [00:17<00:22,  3.42it/s]

 46%|████▌     | 66/143 [00:17<00:24,  3.14it/s]

 47%|████▋     | 67/143 [00:17<00:22,  3.33it/s]

 48%|████▊     | 68/143 [00:18<00:21,  3.42it/s]

 48%|████▊     | 69/143 [00:18<00:23,  3.09it/s]

 49%|████▉     | 70/143 [00:18<00:21,  3.40it/s]

 50%|████▉     | 71/143 [00:19<00:20,  3.45it/s]

 50%|█████     | 72/143 [00:19<00:20,  3.47it/s]

 51%|█████     | 73/143 [00:19<00:20,  3.43it/s]

 52%|█████▏    | 74/143 [00:19<00:20,  3.41it/s]

 52%|█████▏    | 75/143 [00:20<00:20,  3.37it/s]

 53%|█████▎    | 76/143 [00:20<00:19,  3.50it/s]

 54%|█████▍    | 77/143 [00:20<00:18,  3.62it/s]

 55%|█████▍    | 78/143 [00:21<00:16,  3.93it/s]

 55%|█████▌    | 79/143 [00:21<00:16,  3.94it/s]

 56%|█████▌    | 80/143 [00:21<00:18,  3.38it/s]

 57%|█████▋    | 81/143 [00:21<00:16,  3.72it/s]

 57%|█████▋    | 82/143 [00:22<00:17,  3.39it/s]

 58%|█████▊    | 83/143 [00:22<00:18,  3.28it/s]

 59%|█████▊    | 84/143 [00:22<00:17,  3.30it/s]

 59%|█████▉    | 85/143 [00:23<00:16,  3.47it/s]

 60%|██████    | 86/143 [00:23<00:14,  3.99it/s]

 61%|██████    | 87/143 [00:23<00:13,  4.08it/s]

 62%|██████▏   | 88/143 [00:23<00:13,  3.94it/s]

 62%|██████▏   | 89/143 [00:24<00:14,  3.81it/s]

 63%|██████▎   | 90/143 [00:24<00:13,  3.84it/s]

 64%|██████▎   | 91/143 [00:24<00:12,  4.12it/s]

 64%|██████▍   | 92/143 [00:24<00:12,  3.97it/s]

 65%|██████▌   | 93/143 [00:25<00:12,  3.97it/s]

 66%|██████▌   | 94/143 [00:25<00:13,  3.68it/s]

 66%|██████▋   | 95/143 [00:25<00:14,  3.30it/s]

 67%|██████▋   | 96/143 [00:25<00:13,  3.53it/s]

 68%|██████▊   | 97/143 [00:26<00:11,  3.86it/s]

 69%|██████▊   | 98/143 [00:26<00:11,  3.98it/s]

 69%|██████▉   | 99/143 [00:26<00:11,  3.78it/s]

 70%|██████▉   | 100/143 [00:27<00:12,  3.52it/s]

 71%|███████   | 101/143 [00:27<00:11,  3.76it/s]

 71%|███████▏  | 102/143 [00:27<00:11,  3.72it/s]

 72%|███████▏  | 103/143 [00:27<00:11,  3.54it/s]

 73%|███████▎  | 104/143 [00:28<00:10,  3.77it/s]

 73%|███████▎  | 105/143 [00:28<00:10,  3.69it/s]

 74%|███████▍  | 106/143 [00:28<00:10,  3.64it/s]

 75%|███████▍  | 107/143 [00:28<00:09,  3.79it/s]

 76%|███████▌  | 108/143 [00:29<00:08,  3.93it/s]

 76%|███████▌  | 109/143 [00:29<00:08,  3.94it/s]

 77%|███████▋  | 110/143 [00:29<00:08,  4.04it/s]

 78%|███████▊  | 111/143 [00:29<00:08,  3.67it/s]

 78%|███████▊  | 112/143 [00:30<00:08,  3.82it/s]

 79%|███████▉  | 113/143 [00:30<00:08,  3.33it/s]

 80%|███████▉  | 114/143 [00:30<00:08,  3.50it/s]

 80%|████████  | 115/143 [00:31<00:08,  3.50it/s]

 81%|████████  | 116/143 [00:31<00:07,  3.70it/s]

 82%|████████▏ | 117/143 [00:31<00:07,  3.54it/s]

 83%|████████▎ | 118/143 [00:31<00:06,  3.87it/s]

 83%|████████▎ | 119/143 [00:32<00:05,  4.15it/s]

 84%|████████▍ | 120/143 [00:32<00:05,  3.94it/s]

 85%|████████▍ | 121/143 [00:32<00:05,  3.76it/s]

 85%|████████▌ | 122/143 [00:32<00:05,  4.19it/s]

 86%|████████▌ | 123/143 [00:33<00:05,  3.91it/s]

 87%|████████▋ | 124/143 [00:33<00:06,  2.87it/s]

 87%|████████▋ | 125/143 [00:33<00:05,  3.39it/s]

 88%|████████▊ | 126/143 [00:34<00:04,  3.60it/s]

 89%|████████▉ | 127/143 [00:34<00:04,  3.57it/s]

 90%|████████▉ | 128/143 [00:34<00:04,  3.51it/s]

 90%|█████████ | 129/143 [00:34<00:03,  3.71it/s]

 91%|█████████ | 130/143 [00:35<00:03,  3.89it/s]

 92%|█████████▏| 131/143 [00:35<00:02,  4.04it/s]

 92%|█████████▏| 132/143 [00:35<00:02,  4.02it/s]

 93%|█████████▎| 133/143 [00:35<00:02,  3.54it/s]

 94%|█████████▎| 134/143 [00:36<00:02,  3.52it/s]

 94%|█████████▍| 135/143 [00:36<00:02,  3.41it/s]

 95%|█████████▌| 136/143 [00:36<00:01,  3.63it/s]

 96%|█████████▌| 137/143 [00:37<00:01,  3.48it/s]

 97%|█████████▋| 138/143 [00:37<00:01,  3.48it/s]

 97%|█████████▋| 139/143 [00:37<00:01,  3.60it/s]

 98%|█████████▊| 140/143 [00:37<00:00,  3.92it/s]

 99%|█████████▊| 141/143 [00:38<00:00,  3.84it/s]

 99%|█████████▉| 142/143 [00:38<00:00,  3.73it/s]

100%|██████████| 143/143 [00:38<00:00,  3.70it/s]
                                                    

                                                 
{'eval_loss': '6.924e-06', 'eval_runtime': '39.07', 'eval_samples_per_second': '7.294', 'eval_steps_per_second': '3.66', 'eval_entropy': '1.281', 'eval_num_tokens': '5.528e+06', 'eval_logits/chosen': '2.244', 'eval_logits/rejected': '-1.159', 'eval_mean_token_accuracy': '0.6635', 'eval_rewards/chosen': '4.315', 'eval_rewards/rejected': '-15.15', 'eval_rewards/accuracies': '1', 'eval_rewards/margins': '19.47', 'eval_logps/chosen': '-68', 'eval_logps/rejected': '-326.6', 'epoch': '1.776'}

 59%|█████▉    | 600/1013 [1:24:34<51:42,  7.51s/it]

100%|██████████| 143/143 [00:38<00:00,  3.70it/s]

                                                 

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:32<00:00, 32.47s/it]
Writing model shards: 100%|██████████| 1/1 [00:32<00:00, 32.47s/it]

 59%|█████▉    | 601/1013 [1:25:55<4:43:21, 41.27s/it]
 59%|█████▉    | 602/1013 [1:26:03<3:33:52, 31.22s/it]
 60%|█████▉    | 603/1013 [1:26:10<2:44:17, 24.04s/it]
 60%|█████▉    | 604/1013 [1:26:17<2:08:43, 18.88s/it]
 60%|█████▉    | 605/1013 [1:26:25<1:45:09, 15.46s/it]
 60%|█████▉    | 606/1013 [1:26:32<1:28:32, 13.05s/it]
 60%|█████▉    | 607/1013 [1:26:39<1:16:14, 11.27s/it]
 60%|██████    | 608/1013 [1:26:47<1:09:32, 10.30s/it]
 60%|██████    | 609/1013 [1:26:55<1:03:31,  9.43s/it]
 60%|██████    | 610/1013 [1:27:02<59:40,  8.88s/it]  
                                                    
{'loss': '3.902e-08', 'grad_norm': '1.305e-05', 'learning_rate': '2.055e-05', 'entropy': '1.272', 'num_tokens': '5.62e+06', 'logits/chosen': '2.32', 'logits/rejected': '-1.133', 'mean_token_accuracy': '0.6726', 'rewards/chosen': '4.585', 'rewards/rejected': '-15.43', 'rewards/accuracies': '1', 'rewards/margins': '20.02', 'logps/chosen': '-60.81', 'logps/rejected': '-332.6', 'epoch': '1.806'}

 60%|██████    | 610/1013 [1:27:02<59:40,  8.88s/it]
 60%|██████    | 611/1013 [1:27:10<56:33,  8.44s/it]
 60%|██████    | 612/1013 [1:27:17<53:51,  8.06s/it]
 61%|██████    | 613/1013 [1:27:24<52:41,  7.90s/it]
 61%|██████    | 614/1013 [1:27:32<52:15,  7.86s/it]
 61%|██████    | 615/1013 [1:27:40<51:09,  7.71s/it]
 61%|██████    | 616/1013 [1:27:46<49:31,  7.48s/it]
 61%|██████    | 617/1013 [1:27:54<49:03,  7.43s/it]
 61%|██████    | 618/1013 [1:28:01<48:26,  7.36s/it]
 61%|██████    | 619/1013 [1:28:09<49:56,  7.60s/it]
 61%|██████    | 620/1013 [1:28:17<49:33,  7.57s/it]
                                                    
{'loss': '1.713e-07', 'grad_norm': '1.633e-05', 'learning_rate': '1.97e-05', 'entropy': '1.265', 'num_tokens': '5.713e+06', 'logits/chosen': '2.259', 'logits/rejected': '-1.05', 'mean_token_accuracy': '0.6619', 'rewards/chosen': '4.14', 'rewards/rejected': '-15.06', 'rewards/accuracies': '1', 'rewards/margins': '19.2', 'logps/chosen': '-66.24', 'logps/rejected': '-322.6', 'epoch': '1.836'}

 61%|██████    | 620/1013 [1:28:17<49:33,  7.57s/it]
 61%|██████▏   | 621/1013 [1:28:24<48:42,  7.46s/it]
 61%|██████▏   | 622/1013 [1:28:32<49:41,  7.62s/it]
 62%|██████▏   | 623/1013 [1:28:39<49:03,  7.55s/it]
 62%|██████▏   | 624/1013 [1:28:47<48:42,  7.51s/it]
 62%|██████▏   | 625/1013 [1:28:54<48:36,  7.52s/it]
 62%|██████▏   | 626/1013 [1:29:02<48:35,  7.53s/it]
 62%|██████▏   | 627/1013 [1:29:09<48:04,  7.47s/it]
 62%|██████▏   | 628/1013 [1:29:16<47:44,  7.44s/it]
 62%|██████▏   | 629/1013 [1:29:24<46:58,  7.34s/it]
 62%|██████▏   | 630/1013 [1:29:31<46:46,  7.33s/it]
                                                    
{'loss': '9.81e-08', 'grad_norm': '2.956e-05', 'learning_rate': '1.886e-05', 'entropy': '1.268', 'num_tokens': '5.804e+06', 'logits/chosen': '2.307', 'logits/rejected': '-1.197', 'mean_token_accuracy': '0.6677', 'rewards/chosen': '4.233', 'rewards/rejected': '-15.13', 'rewards/accuracies': '1', 'rewards/margins': '19.36', 'logps/chosen': '-65.75', 'logps/rejected': '-326.6', 'epoch': '1.865'}

 62%|██████▏   | 630/1013 [1:29:31<46:46,  7.33s/it]
 62%|██████▏   | 631/1013 [1:29:38<46:51,  7.36s/it]
 62%|██████▏   | 632/1013 [1:29:46<47:20,  7.46s/it]
 62%|██████▏   | 633/1013 [1:29:53<45:39,  7.21s/it]
 63%|██████▎   | 634/1013 [1:30:00<46:43,  7.40s/it]
 63%|██████▎   | 635/1013 [1:30:08<46:17,  7.35s/it]
 63%|██████▎   | 636/1013 [1:30:15<46:39,  7.43s/it]
 63%|██████▎   | 637/1013 [1:30:23<47:14,  7.54s/it]
 63%|██████▎   | 638/1013 [1:30:30<46:28,  7.44s/it]
 63%|██████▎   | 639/1013 [1:30:37<45:29,  7.30s/it]
 63%|██████▎   | 640/1013 [1:30:45<45:51,  7.38s/it]
                                                    
{'loss': '5.206e-08', 'grad_norm': '1.371e-06', 'learning_rate': '1.803e-05', 'entropy': '1.274', 'num_tokens': '5.897e+06', 'logits/chosen': '2.277', 'logits/rejected': '-1.106', 'mean_token_accuracy': '0.6642', 'rewards/chosen': '4.304', 'rewards/rejected': '-15.57', 'rewards/accuracies': '1', 'rewards/margins': '19.88', 'logps/chosen': '-64.37', 'logps/rejected': '-329.2', 'epoch': '1.895'}

 63%|██████▎   | 640/1013 [1:30:45<45:51,  7.38s/it]
 63%|██████▎   | 641/1013 [1:30:53<46:26,  7.49s/it]
 63%|██████▎   | 642/1013 [1:31:00<45:29,  7.36s/it]
 63%|██████▎   | 643/1013 [1:31:07<45:40,  7.41s/it]
 64%|██████▎   | 644/1013 [1:31:14<44:56,  7.31s/it]
 64%|██████▎   | 645/1013 [1:31:22<45:21,  7.39s/it]
 64%|██████▍   | 646/1013 [1:31:29<45:33,  7.45s/it]
 64%|██████▍   | 647/1013 [1:31:37<45:24,  7.44s/it]
 64%|██████▍   | 648/1013 [1:31:45<46:21,  7.62s/it]
 64%|██████▍   | 649/1013 [1:31:52<45:51,  7.56s/it]
 64%|██████▍   | 650/1013 [1:32:00<45:20,  7.49s/it]
                                                    
{'loss': '1.451e-07', 'grad_norm': '5.102e-05', 'learning_rate': '1.721e-05', 'entropy': '1.276', 'num_tokens': '5.99e+06', 'logits/chosen': '2.327', 'logits/rejected': '-1.143', 'mean_token_accuracy': '0.6677', 'rewards/chosen': '4.312', 'rewards/rejected': '-15.41', 'rewards/accuracies': '1', 'rewards/margins': '19.72', 'logps/chosen': '-66.03', 'logps/rejected': '-332.1', 'epoch': '1.924'}

 64%|██████▍   | 650/1013 [1:32:00<45:20,  7.49s/it]
 64%|██████▍   | 651/1013 [1:32:07<45:42,  7.58s/it]
 64%|██████▍   | 652/1013 [1:32:15<46:19,  7.70s/it]
 64%|██████▍   | 653/1013 [1:32:22<45:03,  7.51s/it]
 65%|██████▍   | 654/1013 [1:32:30<44:46,  7.48s/it]
 65%|██████▍   | 655/1013 [1:32:37<44:36,  7.48s/it]
 65%|██████▍   | 656/1013 [1:32:44<43:44,  7.35s/it]
 65%|██████▍   | 657/1013 [1:32:52<43:47,  7.38s/it]
 65%|██████▍   | 658/1013 [1:33:00<44:42,  7.56s/it]
 65%|██████▌   | 659/1013 [1:33:08<44:55,  7.61s/it]
 65%|██████▌   | 660/1013 [1:33:15<44:50,  7.62s/it]
                                                    
{'loss': '1.29e-07', 'grad_norm': '2.456e-05', 'learning_rate': '1.64e-05', 'entropy': '1.27', 'num_tokens': '6.084e+06', 'logits/chosen': '2.289', 'logits/rejected': '-1.128', 'mean_token_accuracy': '0.6573', 'rewards/chosen': '4.226', 'rewards/rejected': '-15.23', 'rewards/accuracies': '1', 'rewards/margins': '19.46', 'logps/chosen': '-64.96', 'logps/rejected': '-326.5', 'epoch': '1.954'}

 65%|██████▌   | 660/1013 [1:33:15<44:50,  7.62s/it]
 65%|██████▌   | 661/1013 [1:33:23<44:47,  7.63s/it]
 65%|██████▌   | 662/1013 [1:33:30<43:37,  7.46s/it]
 65%|██████▌   | 663/1013 [1:33:37<42:59,  7.37s/it]
 66%|██████▌   | 664/1013 [1:33:45<43:29,  7.48s/it]
 66%|██████▌   | 665/1013 [1:33:52<42:47,  7.38s/it]
 66%|██████▌   | 666/1013 [1:33:59<42:40,  7.38s/it]
 66%|██████▌   | 667/1013 [1:34:06<42:07,  7.30s/it]
 66%|██████▌   | 668/1013 [1:34:15<43:42,  7.60s/it]
 66%|██████▌   | 669/1013 [1:34:22<43:47,  7.64s/it]
 66%|██████▌   | 670/1013 [1:34:30<42:57,  7.52s/it]
                                                    
{'loss': '2.275e-07', 'grad_norm': '1.907e-06', 'learning_rate': '1.559e-05', 'entropy': '1.264', 'num_tokens': '6.177e+06', 'logits/chosen': '2.304', 'logits/rejected': '-1.173', 'mean_token_accuracy': '0.6636', 'rewards/chosen': '4.288', 'rewards/rejected': '-15.12', 'rewards/accuracies': '1', 'rewards/margins': '19.41', 'logps/chosen': '-65.31', 'logps/rejected': '-323.9', 'epoch': '1.984'}

 66%|██████▌   | 670/1013 [1:34:30<42:57,  7.52s/it]
 66%|██████▌   | 671/1013 [1:34:37<42:55,  7.53s/it]
 66%|██████▋   | 672/1013 [1:34:44<41:54,  7.37s/it]
 66%|██████▋   | 673/1013 [1:34:51<41:17,  7.29s/it]
 67%|██████▋   | 674/1013 [1:34:59<41:59,  7.43s/it]
 67%|██████▋   | 675/1013 [1:35:06<41:40,  7.40s/it]
 67%|██████▋   | 676/1013 [1:35:10<35:28,  6.32s/it]
 67%|██████▋   | 677/1013 [1:35:18<38:25,  6.86s/it]
 67%|██████▋   | 678/1013 [1:35:26<39:19,  7.04s/it]
 67%|██████▋   | 679/1013 [1:35:33<39:02,  7.01s/it]
 67%|██████▋   | 680/1013 [1:35:39<38:21,  6.91s/it]
                                                    
{'loss': '3.613e-07', 'grad_norm': '0.0002556', 'learning_rate': '1.48e-05', 'entropy': '1.278', 'num_tokens': '6.263e+06', 'logits/chosen': '2.397', 'logits/rejected': '-1.121', 'mean_token_accuracy': '0.6631', 'rewards/chosen': '4.139', 'rewards/rejected': '-15.08', 'rewards/accuracies': '1', 'rewards/margins': '19.22', 'logps/chosen': '-61.41', 'logps/rejected': '-324.1', 'epoch': '2.012'}

 67%|██████▋   | 680/1013 [1:35:39<38:21,  6.91s/it]
 67%|██████▋   | 681/1013 [1:35:48<40:11,  7.26s/it]
 67%|██████▋   | 682/1013 [1:35:55<40:02,  7.26s/it]
 67%|██████▋   | 683/1013 [1:36:02<39:41,  7.22s/it]
 68%|██████▊   | 684/1013 [1:36:10<40:34,  7.40s/it]
 68%|██████▊   | 685/1013 [1:36:17<40:20,  7.38s/it]
 68%|██████▊   | 686/1013 [1:36:25<40:25,  7.42s/it]
 68%|██████▊   | 687/1013 [1:36:32<39:35,  7.29s/it]
 68%|██████▊   | 688/1013 [1:36:39<40:27,  7.47s/it]
 68%|██████▊   | 689/1013 [1:36:47<40:34,  7.51s/it]
 68%|██████▊   | 690/1013 [1:36:55<40:36,  7.54s/it]
                                                    
{'loss': '7.074e-08', 'grad_norm': '4.489e-07', 'learning_rate': '1.402e-05', 'entropy': '1.267', 'num_tokens': '6.358e+06', 'logits/chosen': '2.315', 'logits/rejected': '-1.152', 'mean_token_accuracy': '0.6725', 'rewards/chosen': '4.411', 'rewards/rejected': '-15.46', 'rewards/accuracies': '1', 'rewards/margins': '19.87', 'logps/chosen': '-62.28', 'logps/rejected': '-331.9', 'epoch': '2.041'}

 68%|██████▊   | 690/1013 [1:36:55<40:36,  7.54s/it]
 68%|██████▊   | 691/1013 [1:37:02<40:07,  7.48s/it]
 68%|██████▊   | 692/1013 [1:37:09<39:40,  7.42s/it]
 68%|██████▊   | 693/1013 [1:37:17<40:15,  7.55s/it]
 69%|██████▊   | 694/1013 [1:37:24<38:46,  7.29s/it]
 69%|██████▊   | 695/1013 [1:37:31<39:11,  7.39s/it]
 69%|██████▊   | 696/1013 [1:37:39<39:07,  7.41s/it]
 69%|██████▉   | 697/1013 [1:37:46<38:50,  7.37s/it]
 69%|██████▉   | 698/1013 [1:37:54<38:37,  7.36s/it]
 69%|██████▉   | 699/1013 [1:38:01<38:34,  7.37s/it]
 69%|██████▉   | 700/1013 [1:38:09<39:20,  7.54s/it]
                                                    
{'loss': '2.164e-07', 'grad_norm': '0.0002995', 'learning_rate': '1.325e-05', 'entropy': '1.267', 'num_tokens': '6.45e+06', 'logits/chosen': '2.292', 'logits/rejected': '-1.126', 'mean_token_accuracy': '0.6636', 'rewards/chosen': '4.283', 'rewards/rejected': '-15.47', 'rewards/accuracies': '1', 'rewards/margins': '19.75', 'logps/chosen': '-65.86', 'logps/rejected': '-331.4', 'epoch': '2.071'}

 69%|██████▉   | 700/1013 [1:38:09<39:20,  7.54s/it]

  0%|          | 0/143 [00:00<?, ?it/s]

  1%|▏         | 2/143 [00:00<00:22,  6.23it/s]

  2%|▏         | 3/143 [00:00<00:30,  4.60it/s]

  3%|▎         | 4/143 [00:00<00:32,  4.32it/s]

  3%|▎         | 5/143 [00:01<00:34,  3.95it/s]

  4%|▍         | 6/143 [00:01<00:33,  4.06it/s]

  5%|▍         | 7/143 [00:01<00:31,  4.27it/s]

  6%|▌         | 8/143 [00:01<00:33,  4.02it/s]

  6%|▋         | 9/143 [00:02<00:31,  4.27it/s]

  7%|▋         | 10/143 [00:02<00:32,  4.16it/s]

  8%|▊         | 11/143 [00:02<00:31,  4.18it/s]

  8%|▊         | 12/143 [00:02<00:33,  3.90it/s]

  9%|▉         | 13/143 [00:03<00:34,  3.81it/s]

 10%|▉         | 14/143 [00:03<00:35,  3.61it/s]

 10%|█         | 15/143 [00:03<00:35,  3.62it/s]

 11%|█         | 16/143 [00:03<00:32,  3.94it/s]

 12%|█▏        | 17/143 [00:04<00:29,  4.21it/s]

 13%|█▎        | 18/143 [00:04<00:29,  4.29it/s]

 13%|█▎        | 19/143 [00:04<00:32,  3.82it/s]

 14%|█▍        | 20/143 [00:05<00:33,  3.62it/s]

 15%|█▍        | 21/143 [00:05<00:32,  3.70it/s]

 15%|█▌        | 22/143 [00:05<00:33,  3.64it/s]

 16%|█▌        | 23/143 [00:05<00:32,  3.72it/s]

 17%|█▋        | 24/143 [00:06<00:32,  3.67it/s]

 17%|█▋        | 25/143 [00:06<00:30,  3.82it/s]

 18%|█▊        | 26/143 [00:06<00:29,  3.98it/s]

 19%|█▉        | 27/143 [00:06<00:32,  3.54it/s]

 20%|█▉        | 28/143 [00:07<00:33,  3.44it/s]

 20%|██        | 29/143 [00:07<00:32,  3.46it/s]

 21%|██        | 30/143 [00:07<00:33,  3.39it/s]

 22%|██▏       | 31/143 [00:08<00:32,  3.44it/s]

 22%|██▏       | 32/143 [00:08<00:30,  3.64it/s]

 23%|██▎       | 33/143 [00:08<00:28,  3.85it/s]

 24%|██▍       | 34/143 [00:08<00:28,  3.88it/s]

 24%|██▍       | 35/143 [00:09<00:28,  3.76it/s]

 25%|██▌       | 36/143 [00:09<00:27,  3.94it/s]

 26%|██▌       | 37/143 [00:09<00:26,  3.95it/s]

 27%|██▋       | 38/143 [00:09<00:26,  3.94it/s]

 27%|██▋       | 39/143 [00:10<00:28,  3.69it/s]

 28%|██▊       | 40/143 [00:10<00:26,  3.89it/s]

 29%|██▊       | 41/143 [00:10<00:26,  3.90it/s]

 29%|██▉       | 42/143 [00:10<00:27,  3.67it/s]

 30%|███       | 43/143 [00:11<00:27,  3.60it/s]

 31%|███       | 44/143 [00:11<00:26,  3.68it/s]

 31%|███▏      | 45/143 [00:11<00:25,  3.84it/s]

 32%|███▏      | 46/143 [00:12<00:31,  3.09it/s]

 33%|███▎      | 47/143 [00:12<00:30,  3.17it/s]

 34%|███▎      | 48/143 [00:12<00:28,  3.36it/s]

 34%|███▍      | 49/143 [00:13<00:27,  3.40it/s]

 35%|███▍      | 50/143 [00:13<00:26,  3.46it/s]

 36%|███▌      | 51/143 [00:13<00:26,  3.48it/s]

 36%|███▋      | 52/143 [00:13<00:25,  3.59it/s]

 37%|███▋      | 53/143 [00:14<00:24,  3.75it/s]

 38%|███▊      | 54/143 [00:14<00:25,  3.50it/s]

 38%|███▊      | 55/143 [00:14<00:24,  3.60it/s]

 39%|███▉      | 56/143 [00:14<00:23,  3.78it/s]

 40%|███▉      | 57/143 [00:15<00:22,  3.81it/s]

 41%|████      | 58/143 [00:15<00:20,  4.10it/s]

 41%|████▏     | 59/143 [00:15<00:19,  4.20it/s]

 42%|████▏     | 60/143 [00:15<00:20,  3.98it/s]

 43%|████▎     | 61/143 [00:16<00:19,  4.23it/s]

 43%|████▎     | 62/143 [00:16<00:22,  3.61it/s]

 44%|████▍     | 63/143 [00:16<00:22,  3.62it/s]

 45%|████▍     | 64/143 [00:17<00:23,  3.42it/s]

 45%|████▌     | 65/143 [00:17<00:22,  3.41it/s]

 46%|████▌     | 66/143 [00:17<00:24,  3.13it/s]

 47%|████▋     | 67/143 [00:17<00:22,  3.32it/s]

 48%|████▊     | 68/143 [00:18<00:22,  3.40it/s]

 48%|████▊     | 69/143 [00:18<00:24,  3.08it/s]

 49%|████▉     | 70/143 [00:18<00:21,  3.38it/s]

 50%|████▉     | 71/143 [00:19<00:20,  3.44it/s]

 50%|█████     | 72/143 [00:19<00:20,  3.47it/s]

 51%|█████     | 73/143 [00:19<00:20,  3.43it/s]

 52%|█████▏    | 74/143 [00:20<00:20,  3.41it/s]

 52%|█████▏    | 75/143 [00:20<00:20,  3.35it/s]

 53%|█████▎    | 76/143 [00:20<00:19,  3.50it/s]

 54%|█████▍    | 77/143 [00:20<00:18,  3.62it/s]

 55%|█████▍    | 78/143 [00:21<00:16,  3.91it/s]

 55%|█████▌    | 79/143 [00:21<00:16,  3.92it/s]

 56%|█████▌    | 80/143 [00:21<00:18,  3.37it/s]

 57%|█████▋    | 81/143 [00:21<00:16,  3.71it/s]

 57%|█████▋    | 82/143 [00:22<00:18,  3.38it/s]

 58%|█████▊    | 83/143 [00:22<00:18,  3.27it/s]

 59%|█████▊    | 84/143 [00:22<00:17,  3.29it/s]

 59%|█████▉    | 85/143 [00:23<00:16,  3.45it/s]

 60%|██████    | 86/143 [00:23<00:14,  3.96it/s]

 61%|██████    | 87/143 [00:23<00:13,  4.04it/s]

 62%|██████▏   | 88/143 [00:23<00:14,  3.92it/s]

 62%|██████▏   | 89/143 [00:24<00:19,  2.76it/s]

 63%|██████▎   | 90/143 [00:24<00:17,  3.03it/s]

 64%|██████▎   | 91/143 [00:24<00:15,  3.43it/s]

 64%|██████▍   | 92/143 [00:25<00:14,  3.48it/s]

 65%|██████▌   | 93/143 [00:25<00:13,  3.60it/s]

 66%|██████▌   | 94/143 [00:25<00:14,  3.45it/s]

 66%|██████▋   | 95/143 [00:26<00:15,  3.17it/s]

 67%|██████▋   | 96/143 [00:26<00:13,  3.42it/s]

 68%|██████▊   | 97/143 [00:26<00:12,  3.77it/s]

 69%|██████▊   | 98/143 [00:26<00:11,  3.91it/s]

 69%|██████▉   | 99/143 [00:27<00:11,  3.74it/s]

 70%|██████▉   | 100/143 [00:27<00:12,  3.49it/s]

 71%|███████   | 101/143 [00:27<00:11,  3.73it/s]

 71%|███████▏  | 102/143 [00:27<00:11,  3.69it/s]

 72%|███████▏  | 103/143 [00:28<00:11,  3.52it/s]

 73%|███████▎  | 104/143 [00:28<00:10,  3.77it/s]

 73%|███████▎  | 105/143 [00:28<00:10,  3.68it/s]

 74%|███████▍  | 106/143 [00:29<00:10,  3.64it/s]

 75%|███████▍  | 107/143 [00:29<00:09,  3.80it/s]

 76%|███████▌  | 108/143 [00:29<00:08,  3.93it/s]

 76%|███████▌  | 109/143 [00:29<00:08,  3.94it/s]

 77%|███████▋  | 110/143 [00:29<00:08,  4.04it/s]

 78%|███████▊  | 111/143 [00:30<00:08,  3.67it/s]

 78%|███████▊  | 112/143 [00:30<00:08,  3.82it/s]

 79%|███████▉  | 113/143 [00:30<00:08,  3.34it/s]

 80%|███████▉  | 114/143 [00:31<00:08,  3.49it/s]

 80%|████████  | 115/143 [00:31<00:08,  3.49it/s]

 81%|████████  | 116/143 [00:31<00:07,  3.69it/s]

 82%|████████▏ | 117/143 [00:32<00:07,  3.54it/s]

 83%|████████▎ | 118/143 [00:32<00:06,  3.87it/s]

 83%|████████▎ | 119/143 [00:32<00:05,  4.16it/s]

 84%|████████▍ | 120/143 [00:32<00:05,  3.94it/s]

 85%|████████▍ | 121/143 [00:33<00:05,  3.76it/s]

 85%|████████▌ | 122/143 [00:33<00:05,  4.19it/s]

 86%|████████▌ | 123/143 [00:33<00:05,  3.92it/s]

 87%|████████▋ | 124/143 [00:33<00:04,  4.01it/s]

 87%|████████▋ | 125/143 [00:33<00:04,  4.43it/s]

 88%|████████▊ | 126/143 [00:34<00:03,  4.37it/s]

 89%|████████▉ | 127/143 [00:34<00:03,  4.06it/s]

 90%|████████▉ | 128/143 [00:34<00:03,  3.84it/s]

 90%|█████████ | 129/143 [00:34<00:03,  3.96it/s]

 91%|█████████ | 130/143 [00:35<00:03,  4.08it/s]

 92%|█████████▏| 131/143 [00:35<00:02,  4.20it/s]

 92%|█████████▏| 132/143 [00:35<00:02,  4.12it/s]

 93%|█████████▎| 133/143 [00:35<00:02,  3.58it/s]

 94%|█████████▎| 134/143 [00:36<00:02,  3.55it/s]

 94%|█████████▍| 135/143 [00:36<00:02,  3.44it/s]

 95%|█████████▌| 136/143 [00:36<00:01,  3.64it/s]

 96%|█████████▌| 137/143 [00:37<00:01,  3.50it/s]

 97%|█████████▋| 138/143 [00:37<00:01,  3.48it/s]

 97%|█████████▋| 139/143 [00:37<00:01,  3.59it/s]

 98%|█████████▊| 140/143 [00:37<00:00,  3.92it/s]

 99%|█████████▊| 141/143 [00:38<00:00,  3.84it/s]

 99%|█████████▉| 142/143 [00:38<00:00,  3.73it/s]

100%|██████████| 143/143 [00:38<00:00,  3.75it/s]
                                                    

                                                 
{'eval_loss': '7.35e-06', 'eval_runtime': '39.13', 'eval_samples_per_second': '7.283', 'eval_steps_per_second': '3.654', 'eval_entropy': '1.28', 'eval_num_tokens': '6.45e+06', 'eval_logits/chosen': '2.243', 'eval_logits/rejected': '-1.159', 'eval_mean_token_accuracy': '0.6639', 'eval_rewards/chosen': '4.313', 'eval_rewards/rejected': '-15.16', 'eval_rewards/accuracies': '1', 'eval_rewards/margins': '19.47', 'eval_logps/chosen': '-68.03', 'eval_logps/rejected': '-326.7', 'epoch': '2.071'}

 69%|██████▉   | 700/1013 [1:38:48<39:20,  7.54s/it]

100%|██████████| 143/143 [00:38<00:00,  3.75it/s]

                                                 

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:34<00:00, 34.80s/it]
Writing model shards: 100%|██████████| 1/1 [00:34<00:00, 34.80s/it]

 69%|██████▉   | 701/1013 [1:40:09<3:34:39, 41.28s/it]
 69%|██████▉   | 702/1013 [1:40:16<2:41:14, 31.11s/it]
 69%|██████▉   | 703/1013 [1:40:23<2:03:10, 23.84s/it]
 69%|██████▉   | 704/1013 [1:40:31<1:37:53, 19.01s/it]
 70%|██████▉   | 705/1013 [1:40:38<1:19:40, 15.52s/it]
 70%|██████▉   | 706/1013 [1:40:45<1:06:20, 12.97s/it]
 70%|██████▉   | 707/1013 [1:40:53<58:17, 11.43s/it]  
 70%|██████▉   | 708/1013 [1:41:01<52:10, 10.26s/it]
 70%|██████▉   | 709/1013 [1:41:08<47:20,  9.34s/it]
 70%|███████   | 710/1013 [1:41:15<44:23,  8.79s/it]
                                                    
{'loss': '2.142e-07', 'grad_norm': '8.821e-05', 'learning_rate': '1.25e-05', 'entropy': '1.273', 'num_tokens': '6.54e+06', 'logits/chosen': '2.256', 'logits/rejected': '-1.122', 'mean_token_accuracy': '0.6676', 'rewards/chosen': '4.438', 'rewards/rejected': '-14.91', 'rewards/accuracies': '1', 'rewards/margins': '19.35', 'logps/chosen': '-61.83', 'logps/rejected': '-323.8', 'epoch': '2.101'}

 70%|███████   | 710/1013 [1:41:15<44:23,  8.79s/it]
 70%|███████   | 711/1013 [1:41:23<42:23,  8.42s/it]
 70%|███████   | 712/1013 [1:41:30<40:54,  8.15s/it]
 70%|███████   | 713/1013 [1:41:38<39:34,  7.92s/it]
 70%|███████   | 714/1013 [1:41:45<38:53,  7.80s/it]
 71%|███████   | 715/1013 [1:41:53<38:05,  7.67s/it]
 71%|███████   | 716/1013 [1:42:00<37:08,  7.50s/it]
 71%|███████   | 717/1013 [1:42:07<36:20,  7.36s/it]
 71%|███████   | 718/1013 [1:42:14<36:12,  7.36s/it]
 71%|███████   | 719/1013 [1:42:21<35:41,  7.28s/it]
 71%|███████   | 720/1013 [1:42:29<35:39,  7.30s/it]
                                                    
{'loss': '7.845e-08', 'grad_norm': '6.771e-05', 'learning_rate': '1.176e-05', 'entropy': '1.271', 'num_tokens': '6.63e+06', 'logits/chosen': '2.269', 'logits/rejected': '-1.176', 'mean_token_accuracy': '0.6651', 'rewards/chosen': '4.375', 'rewards/rejected': '-15.26', 'rewards/accuracies': '1', 'rewards/margins': '19.63', 'logps/chosen': '-65.25', 'logps/rejected': '-325.6', 'epoch': '2.13'}

 71%|███████   | 720/1013 [1:42:29<35:39,  7.30s/it]
 71%|███████   | 721/1013 [1:42:36<35:11,  7.23s/it]
 71%|███████▏  | 722/1013 [1:42:43<35:35,  7.34s/it]
 71%|███████▏  | 723/1013 [1:42:51<36:03,  7.46s/it]
 71%|███████▏  | 724/1013 [1:42:59<36:21,  7.55s/it]
 72%|███████▏  | 725/1013 [1:43:06<35:21,  7.37s/it]
 72%|███████▏  | 726/1013 [1:43:13<35:10,  7.35s/it]
 72%|███████▏  | 727/1013 [1:43:20<35:04,  7.36s/it]
 72%|███████▏  | 728/1013 [1:43:28<35:27,  7.46s/it]
 72%|███████▏  | 729/1013 [1:43:35<34:55,  7.38s/it]
 72%|███████▏  | 730/1013 [1:43:42<34:20,  7.28s/it]
                                                    
{'loss': '6.327e-07', 'grad_norm': '0.000679', 'learning_rate': '1.104e-05', 'entropy': '1.263', 'num_tokens': '6.722e+06', 'logits/chosen': '2.3', 'logits/rejected': '-1.124', 'mean_token_accuracy': '0.6666', 'rewards/chosen': '4.219', 'rewards/rejected': '-15.22', 'rewards/accuracies': '1', 'rewards/margins': '19.43', 'logps/chosen': '-65.31', 'logps/rejected': '-327.7', 'epoch': '2.16'}

 72%|███████▏  | 730/1013 [1:43:42<34:20,  7.28s/it]
 72%|███████▏  | 731/1013 [1:43:50<34:15,  7.29s/it]
 72%|███████▏  | 732/1013 [1:43:57<34:05,  7.28s/it]
 72%|███████▏  | 733/1013 [1:44:04<34:13,  7.34s/it]
 72%|███████▏  | 734/1013 [1:44:12<34:10,  7.35s/it]
 73%|███████▎  | 735/1013 [1:44:19<34:02,  7.35s/it]
 73%|███████▎  | 736/1013 [1:44:27<34:11,  7.41s/it]
 73%|███████▎  | 737/1013 [1:44:34<33:48,  7.35s/it]
 73%|███████▎  | 738/1013 [1:44:41<34:00,  7.42s/it]
 73%|███████▎  | 739/1013 [1:44:49<33:34,  7.35s/it]
 73%|███████▎  | 740/1013 [1:44:56<33:56,  7.46s/it]
                                                    
{'loss': '1.791e-07', 'grad_norm': '9.06e-06', 'learning_rate': '1.033e-05', 'entropy': '1.284', 'num_tokens': '6.813e+06', 'logits/chosen': '2.292', 'logits/rejected': '-1.282', 'mean_token_accuracy': '0.6699', 'rewards/chosen': '4.311', 'rewards/rejected': '-15.31', 'rewards/accuracies': '1', 'rewards/margins': '19.62', 'logps/chosen': '-66.78', 'logps/rejected': '-333.5', 'epoch': '2.19'}

 73%|███████▎  | 740/1013 [1:44:56<33:56,  7.46s/it]
 73%|███████▎  | 741/1013 [1:45:04<34:19,  7.57s/it]
 73%|███████▎  | 742/1013 [1:45:12<34:22,  7.61s/it]
 73%|███████▎  | 743/1013 [1:45:19<33:37,  7.47s/it]
 73%|███████▎  | 744/1013 [1:45:27<33:37,  7.50s/it]
 74%|███████▎  | 745/1013 [1:45:34<32:46,  7.34s/it]
 74%|███████▎  | 746/1013 [1:45:41<32:37,  7.33s/it]
 74%|███████▎  | 747/1013 [1:45:48<32:50,  7.41s/it]
 74%|███████▍  | 748/1013 [1:45:56<32:54,  7.45s/it]
 74%|███████▍  | 749/1013 [1:46:03<32:35,  7.41s/it]
 74%|███████▍  | 750/1013 [1:46:11<32:45,  7.47s/it]
                                                    
{'loss': '3.619e-07', 'grad_norm': '2.825e-05', 'learning_rate': '9.645e-06', 'entropy': '1.283', 'num_tokens': '6.906e+06', 'logits/chosen': '2.296', 'logits/rejected': '-1.138', 'mean_token_accuracy': '0.6705', 'rewards/chosen': '4.348', 'rewards/rejected': '-15.13', 'rewards/accuracies': '1', 'rewards/margins': '19.48', 'logps/chosen': '-65.19', 'logps/rejected': '-333.2', 'epoch': '2.219'}

 74%|███████▍  | 750/1013 [1:46:11<32:45,  7.47s/it]
 74%|███████▍  | 751/1013 [1:46:18<32:36,  7.47s/it]
 74%|███████▍  | 752/1013 [1:46:26<32:11,  7.40s/it]
 74%|███████▍  | 753/1013 [1:46:34<32:43,  7.55s/it]
 74%|███████▍  | 754/1013 [1:46:42<33:08,  7.68s/it]
 75%|███████▍  | 755/1013 [1:46:49<32:52,  7.64s/it]
 75%|███████▍  | 756/1013 [1:46:56<32:12,  7.52s/it]
 75%|███████▍  | 757/1013 [1:47:04<32:37,  7.65s/it]
 75%|███████▍  | 758/1013 [1:47:12<32:32,  7.66s/it]
 75%|███████▍  | 759/1013 [1:47:20<32:51,  7.76s/it]
 75%|███████▌  | 760/1013 [1:47:27<32:03,  7.60s/it]
                                                    
{'loss': '2.773e-07', 'grad_norm': '3.672e-05', 'learning_rate': '8.974e-06', 'entropy': '1.289', 'num_tokens': '7.004e+06', 'logits/chosen': '2.274', 'logits/rejected': '-1.197', 'mean_token_accuracy': '0.6637', 'rewards/chosen': '4.223', 'rewards/rejected': '-15.38', 'rewards/accuracies': '1', 'rewards/margins': '19.61', 'logps/chosen': '-62.73', 'logps/rejected': '-335.8', 'epoch': '2.249'}

 75%|███████▌  | 760/1013 [1:47:27<32:03,  7.60s/it]
 75%|███████▌  | 761/1013 [1:47:34<30:58,  7.38s/it]
 75%|███████▌  | 762/1013 [1:47:41<30:30,  7.29s/it]
 75%|███████▌  | 763/1013 [1:47:48<30:01,  7.21s/it]
 75%|███████▌  | 764/1013 [1:47:55<29:33,  7.12s/it]
 76%|███████▌  | 765/1013 [1:48:03<30:17,  7.33s/it]
 76%|███████▌  | 766/1013 [1:48:10<30:16,  7.35s/it]
 76%|███████▌  | 767/1013 [1:48:18<30:30,  7.44s/it]
 76%|███████▌  | 768/1013 [1:48:25<30:32,  7.48s/it]
 76%|███████▌  | 769/1013 [1:48:33<30:20,  7.46s/it]
 76%|███████▌  | 770/1013 [1:48:40<29:47,  7.35s/it]
                                                    
{'loss': '6.9e-08', 'grad_norm': '6.819e-05', 'learning_rate': '8.323e-06', 'entropy': '1.271', 'num_tokens': '7.092e+06', 'logits/chosen': '2.315', 'logits/rejected': '-1.184', 'mean_token_accuracy': '0.6749', 'rewards/chosen': '4.219', 'rewards/rejected': '-15.19', 'rewards/accuracies': '1', 'rewards/margins': '19.41', 'logps/chosen': '-60.06', 'logps/rejected': '-324.1', 'epoch': '2.279'}

 76%|███████▌  | 770/1013 [1:48:40<29:47,  7.35s/it]
 76%|███████▌  | 771/1013 [1:48:47<29:42,  7.36s/it]
 76%|███████▌  | 772/1013 [1:48:55<29:31,  7.35s/it]
 76%|███████▋  | 773/1013 [1:49:02<29:11,  7.30s/it]
 76%|███████▋  | 774/1013 [1:49:10<29:33,  7.42s/it]
 77%|███████▋  | 775/1013 [1:49:17<29:37,  7.47s/it]
 77%|███████▋  | 776/1013 [1:49:25<29:37,  7.50s/it]
 77%|███████▋  | 777/1013 [1:49:33<29:51,  7.59s/it]
 77%|███████▋  | 778/1013 [1:49:40<29:17,  7.48s/it]
 77%|███████▋  | 779/1013 [1:49:47<28:26,  7.29s/it]
 77%|███████▋  | 780/1013 [1:49:54<28:46,  7.41s/it]
                                                    
{'loss': '1.486e-07', 'grad_norm': '7.677e-05', 'learning_rate': '7.691e-06', 'entropy': '1.28', 'num_tokens': '7.183e+06', 'logits/chosen': '2.255', 'logits/rejected': '-1.151', 'mean_token_accuracy': '0.6621', 'rewards/chosen': '4.272', 'rewards/rejected': '-15.11', 'rewards/accuracies': '1', 'rewards/margins': '19.38', 'logps/chosen': '-63.52', 'logps/rejected': '-325.3', 'epoch': '2.308'}

 77%|███████▋  | 780/1013 [1:49:54<28:46,  7.41s/it]
 77%|███████▋  | 781/1013 [1:50:02<29:00,  7.50s/it]
 77%|███████▋  | 782/1013 [1:50:09<28:25,  7.38s/it]
 77%|███████▋  | 783/1013 [1:50:17<28:37,  7.47s/it]
 77%|███████▋  | 784/1013 [1:50:24<28:34,  7.49s/it]
 77%|███████▋  | 785/1013 [1:50:31<27:58,  7.36s/it]
 78%|███████▊  | 786/1013 [1:50:39<28:30,  7.54s/it]
 78%|███████▊  | 787/1013 [1:50:47<28:03,  7.45s/it]
 78%|███████▊  | 788/1013 [1:50:54<27:46,  7.40s/it]
 78%|███████▊  | 789/1013 [1:51:01<27:27,  7.36s/it]
 78%|███████▊  | 790/1013 [1:51:09<27:23,  7.37s/it]
                                                    
{'loss': '3.456e-07', 'grad_norm': '6.646e-06', 'learning_rate': '7.08e-06', 'entropy': '1.283', 'num_tokens': '7.276e+06', 'logits/chosen': '2.311', 'logits/rejected': '-1.175', 'mean_token_accuracy': '0.6744', 'rewards/chosen': '4.2', 'rewards/rejected': '-15.66', 'rewards/accuracies': '1', 'rewards/margins': '19.86', 'logps/chosen': '-59.46', 'logps/rejected': '-336.6', 'epoch': '2.338'}

 78%|███████▊  | 790/1013 [1:51:09<27:23,  7.37s/it]
 78%|███████▊  | 791/1013 [1:51:16<27:28,  7.43s/it]
 78%|███████▊  | 792/1013 [1:51:23<26:58,  7.32s/it]
 78%|███████▊  | 793/1013 [1:51:31<27:45,  7.57s/it]
 78%|███████▊  | 794/1013 [1:51:39<27:19,  7.48s/it]
 78%|███████▊  | 795/1013 [1:51:46<27:32,  7.58s/it]
 79%|███████▊  | 796/1013 [1:51:54<27:08,  7.51s/it]
 79%|███████▊  | 797/1013 [1:52:02<27:23,  7.61s/it]
 79%|███████▉  | 798/1013 [1:52:09<26:34,  7.42s/it]
 79%|███████▉  | 799/1013 [1:52:16<26:16,  7.37s/it]
 79%|███████▉  | 800/1013 [1:52:23<25:49,  7.28s/it]
                                                    
{'loss': '6.806e-07', 'grad_norm': '0.0001593', 'learning_rate': '6.491e-06', 'entropy': '1.268', 'num_tokens': '7.367e+06', 'logits/chosen': '2.337', 'logits/rejected': '-1.123', 'mean_token_accuracy': '0.6766', 'rewards/chosen': '4.387', 'rewards/rejected': '-15.41', 'rewards/accuracies': '1', 'rewards/margins': '19.8', 'logps/chosen': '-58.35', 'logps/rejected': '-331.7', 'epoch': '2.367'}

 79%|███████▉  | 800/1013 [1:52:23<25:49,  7.28s/it]

  0%|          | 0/143 [00:00<?, ?it/s]

  1%|▏         | 2/143 [00:00<00:22,  6.26it/s]

  2%|▏         | 3/143 [00:00<00:30,  4.62it/s]

  3%|▎         | 4/143 [00:00<00:32,  4.33it/s]

  3%|▎         | 5/143 [00:01<00:34,  3.95it/s]

  4%|▍         | 6/143 [00:01<00:33,  4.05it/s]

  5%|▍         | 7/143 [00:01<00:31,  4.27it/s]

  6%|▌         | 8/143 [00:01<00:33,  4.01it/s]

  6%|▋         | 9/143 [00:02<00:31,  4.27it/s]

  7%|▋         | 10/143 [00:02<00:32,  4.15it/s]

  8%|▊         | 11/143 [00:02<00:31,  4.18it/s]

  8%|▊         | 12/143 [00:02<00:33,  3.91it/s]

  9%|▉         | 13/143 [00:03<00:34,  3.81it/s]

 10%|▉         | 14/143 [00:03<00:35,  3.62it/s]

 10%|█         | 15/143 [00:03<00:35,  3.62it/s]

 11%|█         | 16/143 [00:03<00:32,  3.93it/s]

 12%|█▏        | 17/143 [00:04<00:30,  4.19it/s]

 13%|█▎        | 18/143 [00:04<00:29,  4.27it/s]

 13%|█▎        | 19/143 [00:04<00:32,  3.80it/s]

 14%|█▍        | 20/143 [00:05<00:34,  3.61it/s]

 15%|█▍        | 21/143 [00:05<00:33,  3.69it/s]

 15%|█▌        | 22/143 [00:05<00:33,  3.65it/s]

 16%|█▌        | 23/143 [00:05<00:32,  3.73it/s]

 17%|█▋        | 24/143 [00:06<00:32,  3.67it/s]

 17%|█▋        | 25/143 [00:06<00:30,  3.82it/s]

 18%|█▊        | 26/143 [00:06<00:29,  3.99it/s]

 19%|█▉        | 27/143 [00:06<00:32,  3.55it/s]

 20%|█▉        | 28/143 [00:07<00:33,  3.44it/s]

 20%|██        | 29/143 [00:07<00:32,  3.47it/s]

 21%|██        | 30/143 [00:07<00:33,  3.40it/s]

 22%|██▏       | 31/143 [00:08<00:32,  3.43it/s]

 22%|██▏       | 32/143 [00:08<00:30,  3.63it/s]

 23%|██▎       | 33/143 [00:08<00:28,  3.85it/s]

 24%|██▍       | 34/143 [00:08<00:28,  3.89it/s]

 24%|██▍       | 35/143 [00:09<00:28,  3.78it/s]

 25%|██▌       | 36/143 [00:09<00:27,  3.96it/s]

 26%|██▌       | 37/143 [00:09<00:26,  3.97it/s]

 27%|██▋       | 38/143 [00:09<00:26,  3.94it/s]

 27%|██▋       | 39/143 [00:10<00:28,  3.70it/s]

 28%|██▊       | 40/143 [00:10<00:26,  3.90it/s]

 29%|██▊       | 41/143 [00:10<00:26,  3.91it/s]

 29%|██▉       | 42/143 [00:10<00:27,  3.68it/s]

 30%|███       | 43/143 [00:11<00:27,  3.60it/s]

 31%|███       | 44/143 [00:11<00:26,  3.69it/s]

 31%|███▏      | 45/143 [00:11<00:25,  3.84it/s]

 32%|███▏      | 46/143 [00:12<00:31,  3.09it/s]

 33%|███▎      | 47/143 [00:12<00:30,  3.17it/s]

 34%|███▎      | 48/143 [00:12<00:28,  3.37it/s]

 34%|███▍      | 49/143 [00:12<00:27,  3.40it/s]

 35%|███▍      | 50/143 [00:13<00:26,  3.45it/s]

 36%|███▌      | 51/143 [00:13<00:26,  3.47it/s]

 36%|███▋      | 52/143 [00:13<00:25,  3.59it/s]

 37%|███▋      | 53/143 [00:14<00:23,  3.75it/s]

 38%|███▊      | 54/143 [00:14<00:25,  3.51it/s]

 38%|███▊      | 55/143 [00:14<00:24,  3.62it/s]

 39%|███▉      | 56/143 [00:14<00:22,  3.80it/s]

 40%|███▉      | 57/143 [00:15<00:22,  3.82it/s]

 41%|████      | 58/143 [00:15<00:20,  4.11it/s]

 41%|████▏     | 59/143 [00:15<00:19,  4.22it/s]

 42%|████▏     | 60/143 [00:15<00:20,  4.00it/s]

 43%|████▎     | 61/143 [00:16<00:19,  4.24it/s]

 43%|████▎     | 62/143 [00:16<00:22,  3.61it/s]

 44%|████▍     | 63/143 [00:16<00:22,  3.61it/s]

 45%|████▍     | 64/143 [00:17<00:23,  3.42it/s]

 45%|████▌     | 65/143 [00:17<00:22,  3.41it/s]

 46%|████▌     | 66/143 [00:18<00:32,  2.39it/s]

 47%|████▋     | 67/143 [00:18<00:28,  2.69it/s]

 48%|████▊     | 68/143 [00:18<00:25,  2.93it/s]

 48%|████▊     | 69/143 [00:18<00:26,  2.79it/s]

 49%|████▉     | 70/143 [00:19<00:23,  3.13it/s]

 50%|████▉     | 71/143 [00:19<00:22,  3.25it/s]

 50%|█████     | 72/143 [00:19<00:21,  3.34it/s]

 51%|█████     | 73/143 [00:20<00:20,  3.34it/s]

 52%|█████▏    | 74/143 [00:20<00:20,  3.35it/s]

 52%|█████▏    | 75/143 [00:20<00:20,  3.31it/s]

 53%|█████▎    | 76/143 [00:20<00:19,  3.47it/s]

 54%|█████▍    | 77/143 [00:21<00:18,  3.60it/s]

 55%|█████▍    | 78/143 [00:21<00:16,  3.90it/s]

 55%|█████▌    | 79/143 [00:21<00:16,  3.91it/s]

 56%|█████▌    | 80/143 [00:22<00:18,  3.37it/s]

 57%|█████▋    | 81/143 [00:22<00:16,  3.70it/s]

 57%|█████▋    | 82/143 [00:22<00:18,  3.37it/s]

 58%|█████▊    | 83/143 [00:22<00:18,  3.27it/s]

 59%|█████▊    | 84/143 [00:23<00:17,  3.29it/s]

 59%|█████▉    | 85/143 [00:23<00:16,  3.46it/s]

 60%|██████    | 86/143 [00:23<00:14,  3.99it/s]

 61%|██████    | 87/143 [00:23<00:13,  4.08it/s]

 62%|██████▏   | 88/143 [00:24<00:14,  3.92it/s]

 62%|██████▏   | 89/143 [00:24<00:14,  3.79it/s]

 63%|██████▎   | 90/143 [00:24<00:13,  3.83it/s]

 64%|██████▎   | 91/143 [00:24<00:12,  4.12it/s]

 64%|██████▍   | 92/143 [00:25<00:12,  3.96it/s]

 65%|██████▌   | 93/143 [00:25<00:12,  3.96it/s]

 66%|██████▌   | 94/143 [00:25<00:13,  3.67it/s]

 66%|██████▋   | 95/143 [00:26<00:14,  3.29it/s]

 67%|██████▋   | 96/143 [00:26<00:13,  3.53it/s]

 68%|██████▊   | 97/143 [00:26<00:11,  3.86it/s]

 69%|██████▊   | 98/143 [00:26<00:11,  3.97it/s]

 69%|██████▉   | 99/143 [00:27<00:11,  3.77it/s]

 70%|██████▉   | 100/143 [00:27<00:12,  3.52it/s]

 71%|███████   | 101/143 [00:27<00:11,  3.77it/s]

 71%|███████▏  | 102/143 [00:27<00:11,  3.71it/s]

 72%|███████▏  | 103/143 [00:28<00:11,  3.53it/s]

 73%|███████▎  | 104/143 [00:28<00:10,  3.77it/s]

 73%|███████▎  | 105/143 [00:28<00:10,  3.69it/s]

 74%|███████▍  | 106/143 [00:28<00:10,  3.65it/s]

 75%|███████▍  | 107/143 [00:29<00:09,  3.80it/s]

 76%|███████▌  | 108/143 [00:29<00:08,  3.93it/s]

 76%|███████▌  | 109/143 [00:29<00:08,  3.94it/s]

 77%|███████▋  | 110/143 [00:29<00:08,  4.03it/s]

 78%|███████▊  | 111/143 [00:30<00:08,  3.67it/s]

 78%|███████▊  | 112/143 [00:30<00:08,  3.81it/s]

 79%|███████▉  | 113/143 [00:30<00:09,  3.33it/s]

 80%|███████▉  | 114/143 [00:31<00:08,  3.49it/s]

 80%|████████  | 115/143 [00:31<00:08,  3.48it/s]

 81%|████████  | 116/143 [00:31<00:07,  3.67it/s]

 82%|████████▏ | 117/143 [00:32<00:07,  3.52it/s]

 83%|████████▎ | 118/143 [00:32<00:06,  3.85it/s]

 83%|████████▎ | 119/143 [00:32<00:05,  4.14it/s]

 84%|████████▍ | 120/143 [00:32<00:05,  3.92it/s]

 85%|████████▍ | 121/143 [00:32<00:05,  3.75it/s]

 85%|████████▌ | 122/143 [00:33<00:05,  4.18it/s]

 86%|████████▌ | 123/143 [00:33<00:05,  3.91it/s]

 87%|████████▋ | 124/143 [00:33<00:06,  3.08it/s]

 87%|████████▋ | 125/143 [00:34<00:05,  3.58it/s]

 88%|████████▊ | 126/143 [00:34<00:04,  3.75it/s]

 89%|████████▉ | 127/143 [00:34<00:04,  3.67it/s]

 90%|████████▉ | 128/143 [00:34<00:04,  3.58it/s]

 90%|█████████ | 129/143 [00:35<00:03,  3.75it/s]

 91%|█████████ | 130/143 [00:35<00:03,  3.93it/s]

 92%|█████████▏| 131/143 [00:35<00:02,  4.07it/s]

 92%|█████████▏| 132/143 [00:35<00:02,  4.03it/s]

 93%|█████████▎| 133/143 [00:36<00:02,  3.53it/s]

 94%|█████████▎| 134/143 [00:36<00:02,  3.52it/s]

 94%|█████████▍| 135/143 [00:36<00:02,  3.41it/s]

 95%|█████████▌| 136/143 [00:37<00:01,  3.62it/s]

 96%|█████████▌| 137/143 [00:37<00:01,  3.48it/s]

 97%|█████████▋| 138/143 [00:37<00:01,  3.47it/s]

 97%|█████████▋| 139/143 [00:37<00:01,  3.60it/s]

 98%|█████████▊| 140/143 [00:38<00:00,  3.91it/s]

 99%|█████████▊| 141/143 [00:38<00:00,  3.82it/s]

 99%|█████████▉| 142/143 [00:38<00:00,  3.71it/s]

100%|██████████| 143/143 [00:39<00:00,  3.57it/s]
                                                    

                                                 
{'eval_loss': '7.024e-06', 'eval_runtime': '39.41', 'eval_samples_per_second': '7.231', 'eval_steps_per_second': '3.628', 'eval_entropy': '1.28', 'eval_num_tokens': '7.367e+06', 'eval_logits/chosen': '2.244', 'eval_logits/rejected': '-1.159', 'eval_mean_token_accuracy': '0.6632', 'eval_rewards/chosen': '4.313', 'eval_rewards/rejected': '-15.17', 'eval_rewards/accuracies': '1', 'eval_rewards/margins': '19.48', 'eval_logps/chosen': '-68.02', 'eval_logps/rejected': '-326.7', 'epoch': '2.367'}

 79%|███████▉  | 800/1013 [1:53:02<25:49,  7.28s/it]

100%|██████████| 143/143 [00:39<00:00,  3.57it/s]

                                                 

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:33<00:00, 33.21s/it]
Writing model shards: 100%|██████████| 1/1 [00:33<00:00, 33.22s/it]

 79%|███████▉  | 801/1013 [1:54:27<2:29:15, 42.24s/it]
 79%|███████▉  | 802/1013 [1:54:34<1:51:27, 31.70s/it]
 79%|███████▉  | 803/1013 [1:54:41<1:25:09, 24.33s/it]
 79%|███████▉  | 804/1013 [1:54:48<1:06:33, 19.11s/it]
 79%|███████▉  | 805/1013 [1:54:55<53:26, 15.42s/it]  
 80%|███████▉  | 806/1013 [1:55:02<44:56, 13.03s/it]
 80%|███████▉  | 807/1013 [1:55:09<38:20, 11.17s/it]
 80%|███████▉  | 808/1013 [1:55:16<34:20, 10.05s/it]
 80%|███████▉  | 809/1013 [1:55:23<31:08,  9.16s/it]
 80%|███████▉  | 810/1013 [1:55:31<29:14,  8.64s/it]
                                                    
{'loss': '2.754e-07', 'grad_norm': '0.0001006', 'learning_rate': '5.923e-06', 'entropy': '1.254', 'num_tokens': '7.456e+06', 'logits/chosen': '2.306', 'logits/rejected': '-1.103', 'mean_token_accuracy': '0.6696', 'rewards/chosen': '4.309', 'rewards/rejected': '-14.81', 'rewards/accuracies': '1', 'rewards/margins': '19.12', 'logps/chosen': '-64.73', 'logps/rejected': '-316.8', 'epoch': '2.397'}

 80%|███████▉  | 810/1013 [1:55:31<29:14,  8.64s/it]
 80%|████████  | 811/1013 [1:55:38<27:58,  8.31s/it]
 80%|████████  | 812/1013 [1:55:46<26:42,  7.97s/it]
 80%|████████  | 813/1013 [1:55:53<26:27,  7.94s/it]
 80%|████████  | 814/1013 [1:56:01<25:52,  7.80s/it]
 80%|████████  | 815/1013 [1:56:08<24:52,  7.54s/it]
 81%|████████  | 816/1013 [1:56:16<25:06,  7.65s/it]
 81%|████████  | 817/1013 [1:56:23<24:23,  7.47s/it]
 81%|████████  | 818/1013 [1:56:31<24:36,  7.57s/it]
 81%|████████  | 819/1013 [1:56:38<24:35,  7.61s/it]
 81%|████████  | 820/1013 [1:56:46<24:01,  7.47s/it]
                                                    
{'loss': '9.869e-08', 'grad_norm': '3.844e-06', 'learning_rate': '5.378e-06', 'entropy': '1.269', 'num_tokens': '7.549e+06', 'logits/chosen': '2.276', 'logits/rejected': '-1.221', 'mean_token_accuracy': '0.6663', 'rewards/chosen': '4.394', 'rewards/rejected': '-15.49', 'rewards/accuracies': '1', 'rewards/margins': '19.88', 'logps/chosen': '-64.78', 'logps/rejected': '-330.2', 'epoch': '2.427'}

 81%|████████  | 820/1013 [1:56:46<24:01,  7.47s/it]
 81%|████████  | 821/1013 [1:56:52<23:23,  7.31s/it]
 81%|████████  | 822/1013 [1:57:00<23:28,  7.37s/it]
 81%|████████  | 823/1013 [1:57:07<23:08,  7.31s/it]
 81%|████████▏ | 824/1013 [1:57:14<22:36,  7.18s/it]
 81%|████████▏ | 825/1013 [1:57:21<22:05,  7.05s/it]
 82%|████████▏ | 826/1013 [1:57:28<21:54,  7.03s/it]
 82%|████████▏ | 827/1013 [1:57:36<22:35,  7.29s/it]
 82%|████████▏ | 828/1013 [1:57:43<22:38,  7.34s/it]
 82%|████████▏ | 829/1013 [1:57:51<22:39,  7.39s/it]
 82%|████████▏ | 830/1013 [1:57:58<22:12,  7.28s/it]
                                                    
{'loss': '1.597e-07', 'grad_norm': '1.574e-05', 'learning_rate': '4.856e-06', 'entropy': '1.276', 'num_tokens': '7.637e+06', 'logits/chosen': '2.261', 'logits/rejected': '-1.167', 'mean_token_accuracy': '0.6732', 'rewards/chosen': '4.313', 'rewards/rejected': '-15.15', 'rewards/accuracies': '1', 'rewards/margins': '19.46', 'logps/chosen': '-63.7', 'logps/rejected': '-323.2', 'epoch': '2.456'}

 82%|████████▏ | 830/1013 [1:57:58<22:12,  7.28s/it]
 82%|████████▏ | 831/1013 [1:58:05<21:58,  7.24s/it]
 82%|████████▏ | 832/1013 [1:58:12<21:41,  7.19s/it]
 82%|████████▏ | 833/1013 [1:58:19<21:24,  7.14s/it]
 82%|████████▏ | 834/1013 [1:58:26<21:29,  7.20s/it]
 82%|████████▏ | 835/1013 [1:58:34<21:39,  7.30s/it]
 83%|████████▎ | 836/1013 [1:58:41<21:13,  7.19s/it]
 83%|████████▎ | 837/1013 [1:58:48<21:12,  7.23s/it]
 83%|████████▎ | 838/1013 [1:58:56<21:21,  7.32s/it]
 83%|████████▎ | 839/1013 [1:59:03<21:24,  7.38s/it]
 83%|████████▎ | 840/1013 [1:59:10<21:14,  7.37s/it]
                                                    
{'loss': '1.426e-07', 'grad_norm': '0.0002842', 'learning_rate': '4.358e-06', 'entropy': '1.261', 'num_tokens': '7.726e+06', 'logits/chosen': '2.29', 'logits/rejected': '-1.247', 'mean_token_accuracy': '0.6746', 'rewards/chosen': '4.292', 'rewards/rejected': '-15.13', 'rewards/accuracies': '1', 'rewards/margins': '19.42', 'logps/chosen': '-62.79', 'logps/rejected': '-325', 'epoch': '2.486'}

 83%|████████▎ | 840/1013 [1:59:10<21:14,  7.37s/it]
 83%|████████▎ | 841/1013 [1:59:18<21:16,  7.42s/it]
 83%|████████▎ | 842/1013 [1:59:25<21:16,  7.46s/it]
 83%|████████▎ | 843/1013 [1:59:34<21:44,  7.67s/it]
 83%|████████▎ | 844/1013 [1:59:41<21:28,  7.63s/it]
 83%|████████▎ | 845/1013 [1:59:49<21:37,  7.72s/it]
 84%|████████▎ | 846/1013 [1:59:57<21:18,  7.65s/it]
 84%|████████▎ | 847/1013 [2:00:04<20:45,  7.50s/it]
 84%|████████▎ | 848/1013 [2:00:12<20:57,  7.62s/it]
 84%|████████▍ | 849/1013 [2:00:19<20:26,  7.48s/it]
 84%|████████▍ | 850/1013 [2:00:26<20:13,  7.45s/it]
                                                    
{'loss': '1.493e-07', 'grad_norm': '0.0002613', 'learning_rate': '3.884e-06', 'entropy': '1.27', 'num_tokens': '7.822e+06', 'logits/chosen': '2.386', 'logits/rejected': '-1.125', 'mean_token_accuracy': '0.679', 'rewards/chosen': '4.364', 'rewards/rejected': '-15.14', 'rewards/accuracies': '1', 'rewards/margins': '19.51', 'logps/chosen': '-61.92', 'logps/rejected': '-327.4', 'epoch': '2.516'}

 84%|████████▍ | 850/1013 [2:00:26<20:13,  7.45s/it]
 84%|████████▍ | 851/1013 [2:00:33<19:45,  7.32s/it]
 84%|████████▍ | 852/1013 [2:00:41<19:47,  7.38s/it]
 84%|████████▍ | 853/1013 [2:00:48<19:57,  7.48s/it]
 84%|████████▍ | 854/1013 [2:00:56<20:10,  7.62s/it]
 84%|████████▍ | 855/1013 [2:01:04<19:43,  7.49s/it]
 85%|████████▍ | 856/1013 [2:01:11<19:43,  7.54s/it]
 85%|████████▍ | 857/1013 [2:01:18<19:16,  7.41s/it]
 85%|████████▍ | 858/1013 [2:01:26<19:19,  7.48s/it]
 85%|████████▍ | 859/1013 [2:01:33<19:06,  7.44s/it]
 85%|████████▍ | 860/1013 [2:01:41<18:56,  7.43s/it]
                                                    
{'loss': '1.422e-07', 'grad_norm': '4.506e-05', 'learning_rate': '3.436e-06', 'entropy': '1.25', 'num_tokens': '7.916e+06', 'logits/chosen': '2.299', 'logits/rejected': '-1.106', 'mean_token_accuracy': '0.6769', 'rewards/chosen': '4.385', 'rewards/rejected': '-15.66', 'rewards/accuracies': '1', 'rewards/margins': '20.05', 'logps/chosen': '-65.89', 'logps/rejected': '-330', 'epoch': '2.545'}

 85%|████████▍ | 860/1013 [2:01:41<18:56,  7.43s/it]
 85%|████████▍ | 861/1013 [2:01:48<18:40,  7.37s/it]
 85%|████████▌ | 862/1013 [2:01:55<18:22,  7.30s/it]
 85%|████████▌ | 863/1013 [2:02:02<18:15,  7.30s/it]
 85%|████████▌ | 864/1013 [2:02:10<18:36,  7.49s/it]
 85%|████████▌ | 865/1013 [2:02:18<18:28,  7.49s/it]
 85%|████████▌ | 866/1013 [2:02:25<18:01,  7.36s/it]
 86%|████████▌ | 867/1013 [2:02:32<18:05,  7.43s/it]
 86%|████████▌ | 868/1013 [2:02:40<17:46,  7.35s/it]
 86%|████████▌ | 869/1013 [2:02:47<17:56,  7.48s/it]
 86%|████████▌ | 870/1013 [2:02:54<17:27,  7.32s/it]
                                                    
{'loss': '2.536e-07', 'grad_norm': '2.515e-05', 'learning_rate': '3.013e-06', 'entropy': '1.28', 'num_tokens': '8.007e+06', 'logits/chosen': '2.237', 'logits/rejected': '-1.122', 'mean_token_accuracy': '0.6659', 'rewards/chosen': '4.355', 'rewards/rejected': '-14.86', 'rewards/accuracies': '1', 'rewards/margins': '19.22', 'logps/chosen': '-65.73', 'logps/rejected': '-322.3', 'epoch': '2.575'}

 86%|████████▌ | 870/1013 [2:02:54<17:27,  7.32s/it]
 86%|████████▌ | 871/1013 [2:03:02<17:21,  7.34s/it]
 86%|████████▌ | 872/1013 [2:03:09<17:18,  7.36s/it]
 86%|████████▌ | 873/1013 [2:03:16<17:00,  7.29s/it]
 86%|████████▋ | 874/1013 [2:03:24<17:08,  7.40s/it]
 86%|████████▋ | 875/1013 [2:03:32<17:17,  7.52s/it]
 86%|████████▋ | 876/1013 [2:03:39<17:04,  7.48s/it]
 87%|████████▋ | 877/1013 [2:03:47<17:09,  7.57s/it]
 87%|████████▋ | 878/1013 [2:03:55<17:02,  7.58s/it]
 87%|████████▋ | 879/1013 [2:04:02<16:50,  7.54s/it]
 87%|████████▋ | 880/1013 [2:04:09<16:39,  7.51s/it]
                                                    
{'loss': '2.378e-07', 'grad_norm': '1.174e-05', 'learning_rate': '2.616e-06', 'entropy': '1.276', 'num_tokens': '8.1e+06', 'logits/chosen': '2.28', 'logits/rejected': '-1.132', 'mean_token_accuracy': '0.6662', 'rewards/chosen': '4.331', 'rewards/rejected': '-15.15', 'rewards/accuracies': '1', 'rewards/margins': '19.48', 'logps/chosen': '-61.99', 'logps/rejected': '-326.9', 'epoch': '2.604'}

 87%|████████▋ | 880/1013 [2:04:09<16:39,  7.51s/it]
 87%|████████▋ | 881/1013 [2:04:17<16:37,  7.55s/it]
 87%|████████▋ | 882/1013 [2:04:24<16:20,  7.48s/it]
 87%|████████▋ | 883/1013 [2:04:32<16:27,  7.60s/it]
 87%|████████▋ | 884/1013 [2:04:40<16:08,  7.51s/it]
 87%|████████▋ | 885/1013 [2:04:47<16:05,  7.55s/it]
 87%|████████▋ | 886/1013 [2:04:55<15:56,  7.53s/it]
 88%|████████▊ | 887/1013 [2:05:02<15:39,  7.46s/it]
 88%|████████▊ | 888/1013 [2:05:10<15:40,  7.52s/it]
 88%|████████▊ | 889/1013 [2:05:17<15:37,  7.56s/it]
 88%|████████▊ | 890/1013 [2:05:25<15:31,  7.57s/it]
                                                    
{'loss': '6.257e-08', 'grad_norm': '9.632e-05', 'learning_rate': '2.246e-06', 'entropy': '1.279', 'num_tokens': '8.194e+06', 'logits/chosen': '2.303', 'logits/rejected': '-1.171', 'mean_token_accuracy': '0.6666', 'rewards/chosen': '4.474', 'rewards/rejected': '-15.38', 'rewards/accuracies': '1', 'rewards/margins': '19.85', 'logps/chosen': '-65.01', 'logps/rejected': '-330.7', 'epoch': '2.634'}

 88%|████████▊ | 890/1013 [2:05:25<15:31,  7.57s/it]
 88%|████████▊ | 891/1013 [2:05:32<15:03,  7.41s/it]
 88%|████████▊ | 892/1013 [2:05:39<15:03,  7.46s/it]
 88%|████████▊ | 893/1013 [2:05:47<14:51,  7.43s/it]
 88%|████████▊ | 894/1013 [2:05:54<14:50,  7.48s/it]
 88%|████████▊ | 895/1013 [2:06:03<15:05,  7.68s/it]
 88%|████████▊ | 896/1013 [2:06:10<14:41,  7.53s/it]
 89%|████████▊ | 897/1013 [2:06:17<14:23,  7.44s/it]
 89%|████████▊ | 898/1013 [2:06:24<14:00,  7.31s/it]
 89%|████████▊ | 899/1013 [2:06:32<14:07,  7.43s/it]
 89%|████████▉ | 900/1013 [2:06:39<13:45,  7.31s/it]
                                                    
{'loss': '1.403e-07', 'grad_norm': '2.456e-05', 'learning_rate': '1.903e-06', 'entropy': '1.26', 'num_tokens': '8.286e+06', 'logits/chosen': '2.249', 'logits/rejected': '-1.094', 'mean_token_accuracy': '0.6666', 'rewards/chosen': '4.302', 'rewards/rejected': '-15', 'rewards/accuracies': '1', 'rewards/margins': '19.3', 'logps/chosen': '-64.42', 'logps/rejected': '-319.6', 'epoch': '2.664'}

 89%|████████▉ | 900/1013 [2:06:39<13:45,  7.31s/it]

  0%|          | 0/143 [00:00<?, ?it/s]

  1%|▏         | 2/143 [00:00<00:23,  5.98it/s]

  2%|▏         | 3/143 [00:00<00:30,  4.52it/s]

  3%|▎         | 4/143 [00:00<00:32,  4.28it/s]

  3%|▎         | 5/143 [00:01<00:35,  3.93it/s]

  4%|▍         | 6/143 [00:01<00:33,  4.04it/s]

  5%|▍         | 7/143 [00:01<00:31,  4.26it/s]

  6%|▌         | 8/143 [00:01<00:33,  4.01it/s]

  6%|▋         | 9/143 [00:02<00:31,  4.27it/s]

  7%|▋         | 10/143 [00:02<00:32,  4.15it/s]

  8%|▊         | 11/143 [00:02<00:31,  4.18it/s]

  8%|▊         | 12/143 [00:02<00:33,  3.90it/s]

  9%|▉         | 13/143 [00:03<00:34,  3.82it/s]

 10%|▉         | 14/143 [00:03<00:35,  3.62it/s]

 10%|█         | 15/143 [00:03<00:35,  3.62it/s]

 11%|█         | 16/143 [00:03<00:32,  3.93it/s]

 12%|█▏        | 17/143 [00:04<00:30,  4.19it/s]

 13%|█▎        | 18/143 [00:04<00:29,  4.28it/s]

 13%|█▎        | 19/143 [00:04<00:32,  3.80it/s]

 14%|█▍        | 20/143 [00:05<00:34,  3.61it/s]

 15%|█▍        | 21/143 [00:05<00:33,  3.69it/s]

 15%|█▌        | 22/143 [00:05<00:33,  3.65it/s]

 16%|█▌        | 23/143 [00:05<00:32,  3.72it/s]

 17%|█▋        | 24/143 [00:06<00:32,  3.66it/s]

 17%|█▋        | 25/143 [00:06<00:30,  3.81it/s]

 18%|█▊        | 26/143 [00:06<00:29,  3.97it/s]

 19%|█▉        | 27/143 [00:06<00:32,  3.54it/s]

 20%|█▉        | 28/143 [00:07<00:33,  3.44it/s]

 20%|██        | 29/143 [00:07<00:33,  3.45it/s]

 21%|██        | 30/143 [00:07<00:33,  3.39it/s]

 22%|██▏       | 31/143 [00:08<00:32,  3.43it/s]

 22%|██▏       | 32/143 [00:08<00:30,  3.64it/s]

 23%|██▎       | 33/143 [00:08<00:28,  3.84it/s]

 24%|██▍       | 34/143 [00:08<00:28,  3.88it/s]

 24%|██▍       | 35/143 [00:09<00:28,  3.77it/s]

 25%|██▌       | 36/143 [00:09<00:27,  3.96it/s]

 26%|██▌       | 37/143 [00:09<00:26,  3.96it/s]

 27%|██▋       | 38/143 [00:09<00:26,  3.94it/s]

 27%|██▋       | 39/143 [00:10<00:28,  3.69it/s]

 28%|██▊       | 40/143 [00:10<00:26,  3.89it/s]

 29%|██▊       | 41/143 [00:10<00:26,  3.90it/s]

 29%|██▉       | 42/143 [00:10<00:27,  3.67it/s]

 30%|███       | 43/143 [00:11<00:27,  3.62it/s]

 31%|███       | 44/143 [00:11<00:26,  3.70it/s]

 31%|███▏      | 45/143 [00:11<00:25,  3.86it/s]

 32%|███▏      | 46/143 [00:12<00:31,  3.09it/s]

 33%|███▎      | 47/143 [00:12<00:39,  2.41it/s]

 34%|███▎      | 48/143 [00:13<00:34,  2.73it/s]

 34%|███▍      | 49/143 [00:13<00:32,  2.92it/s]

 35%|███▍      | 50/143 [00:13<00:30,  3.09it/s]

 36%|███▌      | 51/143 [00:13<00:28,  3.21it/s]

 36%|███▋      | 52/143 [00:14<00:26,  3.39it/s]

 37%|███▋      | 53/143 [00:14<00:25,  3.60it/s]

 38%|███▊      | 54/143 [00:14<00:26,  3.42it/s]

 38%|███▊      | 55/143 [00:14<00:24,  3.55it/s]

 39%|███▉      | 56/143 [00:15<00:23,  3.76it/s]

 40%|███▉      | 57/143 [00:15<00:22,  3.79it/s]

 41%|████      | 58/143 [00:15<00:20,  4.06it/s]

 41%|████▏     | 59/143 [00:15<00:20,  4.17it/s]

 42%|████▏     | 60/143 [00:16<00:21,  3.95it/s]

 43%|████▎     | 61/143 [00:16<00:19,  4.20it/s]

 43%|████▎     | 62/143 [00:16<00:22,  3.59it/s]

 44%|████▍     | 63/143 [00:17<00:22,  3.61it/s]

 45%|████▍     | 64/143 [00:17<00:23,  3.41it/s]

 45%|████▌     | 65/143 [00:17<00:22,  3.40it/s]

 46%|████▌     | 66/143 [00:18<00:24,  3.13it/s]

 47%|████▋     | 67/143 [00:18<00:22,  3.31it/s]

 48%|████▊     | 68/143 [00:18<00:22,  3.41it/s]

 48%|████▊     | 69/143 [00:18<00:24,  3.08it/s]

 49%|████▉     | 70/143 [00:19<00:21,  3.38it/s]

 50%|████▉     | 71/143 [00:19<00:20,  3.43it/s]

 50%|█████     | 72/143 [00:19<00:20,  3.47it/s]

 51%|█████     | 73/143 [00:20<00:20,  3.42it/s]

 52%|█████▏    | 74/143 [00:20<00:20,  3.41it/s]

 52%|█████▏    | 75/143 [00:20<00:20,  3.35it/s]

 53%|█████▎    | 76/143 [00:20<00:19,  3.50it/s]

 54%|█████▍    | 77/143 [00:21<00:18,  3.62it/s]

 55%|█████▍    | 78/143 [00:21<00:16,  3.91it/s]

 55%|█████▌    | 79/143 [00:21<00:16,  3.91it/s]

 56%|█████▌    | 80/143 [00:22<00:18,  3.37it/s]

 57%|█████▋    | 81/143 [00:22<00:16,  3.70it/s]

 57%|█████▋    | 82/143 [00:22<00:18,  3.38it/s]

 58%|█████▊    | 83/143 [00:22<00:18,  3.27it/s]

 59%|█████▊    | 84/143 [00:23<00:17,  3.30it/s]

 59%|█████▉    | 85/143 [00:23<00:16,  3.46it/s]

 60%|██████    | 86/143 [00:23<00:14,  3.98it/s]

 61%|██████    | 87/143 [00:23<00:13,  4.07it/s]

 62%|██████▏   | 88/143 [00:24<00:13,  3.94it/s]

 62%|██████▏   | 89/143 [00:24<00:14,  3.79it/s]

 63%|██████▎   | 90/143 [00:24<00:13,  3.83it/s]

 64%|██████▎   | 91/143 [00:24<00:12,  4.11it/s]

 64%|██████▍   | 92/143 [00:25<00:12,  3.96it/s]

 65%|██████▌   | 93/143 [00:25<00:12,  3.96it/s]

 66%|██████▌   | 94/143 [00:25<00:13,  3.67it/s]

 66%|██████▋   | 95/143 [00:26<00:14,  3.29it/s]

 67%|██████▋   | 96/143 [00:26<00:13,  3.53it/s]

 68%|██████▊   | 97/143 [00:26<00:11,  3.85it/s]

 69%|██████▊   | 98/143 [00:26<00:11,  3.96it/s]

 69%|██████▉   | 99/143 [00:27<00:11,  3.76it/s]

 70%|██████▉   | 100/143 [00:27<00:12,  3.52it/s]

 71%|███████   | 101/143 [00:27<00:11,  3.77it/s]

 71%|███████▏  | 102/143 [00:27<00:11,  3.70it/s]

 72%|███████▏  | 103/143 [00:28<00:11,  3.53it/s]

 73%|███████▎  | 104/143 [00:28<00:10,  3.76it/s]

 73%|███████▎  | 105/143 [00:28<00:10,  3.69it/s]

 74%|███████▍  | 106/143 [00:29<00:10,  3.65it/s]

 75%|███████▍  | 107/143 [00:29<00:12,  2.97it/s]

 76%|███████▌  | 108/143 [00:29<00:10,  3.27it/s]

 76%|███████▌  | 109/143 [00:30<00:09,  3.44it/s]

 77%|███████▋  | 110/143 [00:30<00:09,  3.66it/s]

 78%|███████▊  | 111/143 [00:30<00:09,  3.44it/s]

 78%|███████▊  | 112/143 [00:30<00:08,  3.64it/s]

 79%|███████▉  | 113/143 [00:31<00:09,  3.23it/s]

 80%|███████▉  | 114/143 [00:31<00:08,  3.42it/s]

 80%|████████  | 115/143 [00:31<00:08,  3.44it/s]

 81%|████████  | 116/143 [00:31<00:07,  3.64it/s]

 82%|████████▏ | 117/143 [00:32<00:07,  3.50it/s]

 83%|████████▎ | 118/143 [00:32<00:06,  3.83it/s]

 83%|████████▎ | 119/143 [00:32<00:05,  4.12it/s]

 84%|████████▍ | 120/143 [00:32<00:05,  3.90it/s]

 85%|████████▍ | 121/143 [00:33<00:05,  3.73it/s]

 85%|████████▌ | 122/143 [00:33<00:05,  4.17it/s]

 86%|████████▌ | 123/143 [00:33<00:05,  3.91it/s]

 87%|████████▋ | 124/143 [00:33<00:04,  3.99it/s]

 87%|████████▋ | 125/143 [00:34<00:04,  4.41it/s]

 88%|████████▊ | 126/143 [00:34<00:03,  4.35it/s]

 89%|████████▉ | 127/143 [00:34<00:03,  4.04it/s]

 90%|████████▉ | 128/143 [00:34<00:03,  3.83it/s]

 90%|█████████ | 129/143 [00:35<00:03,  3.95it/s]

 91%|█████████ | 130/143 [00:35<00:03,  4.06it/s]

 92%|█████████▏| 131/143 [00:35<00:02,  4.17it/s]

 92%|█████████▏| 132/143 [00:35<00:02,  4.10it/s]

 93%|█████████▎| 133/143 [00:36<00:02,  3.57it/s]

 94%|█████████▎| 134/143 [00:36<00:02,  3.54it/s]

 94%|█████████▍| 135/143 [00:36<00:02,  3.42it/s]

 95%|█████████▌| 136/143 [00:37<00:01,  3.63it/s]

 96%|█████████▌| 137/143 [00:37<00:01,  3.48it/s]

 97%|█████████▋| 138/143 [00:37<00:01,  3.47it/s]

 97%|█████████▋| 139/143 [00:37<00:01,  3.59it/s]

 98%|█████████▊| 140/143 [00:38<00:00,  3.90it/s]

 99%|█████████▊| 141/143 [00:38<00:00,  3.82it/s]

 99%|█████████▉| 142/143 [00:38<00:00,  3.73it/s]

100%|██████████| 143/143 [00:39<00:00,  3.57it/s]
                                                    

                                                 
{'eval_loss': '6.497e-06', 'eval_runtime': '39.45', 'eval_samples_per_second': '7.224', 'eval_steps_per_second': '3.625', 'eval_entropy': '1.28', 'eval_num_tokens': '8.286e+06', 'eval_logits/chosen': '2.245', 'eval_logits/rejected': '-1.16', 'eval_mean_token_accuracy': '0.6639', 'eval_rewards/chosen': '4.317', 'eval_rewards/rejected': '-15.17', 'eval_rewards/accuracies': '1', 'eval_rewards/margins': '19.49', 'eval_logps/chosen': '-67.98', 'eval_logps/rejected': '-326.8', 'epoch': '2.664'}

 89%|████████▉ | 900/1013 [2:07:18<13:45,  7.31s/it]

100%|██████████| 143/143 [00:39<00:00,  3.57it/s]

                                                 

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:33<00:00, 33.26s/it]
Writing model shards: 100%|██████████| 1/1 [00:33<00:00, 33.26s/it]

 89%|████████▉ | 901/1013 [2:08:37<1:15:36, 40.50s/it]
 89%|████████▉ | 902/1013 [2:08:44<56:33, 30.57s/it]  
 89%|████████▉ | 903/1013 [2:08:52<43:34, 23.77s/it]
 89%|████████▉ | 904/1013 [2:08:59<34:16, 18.87s/it]
 89%|████████▉ | 905/1013 [2:09:07<27:49, 15.46s/it]
 89%|████████▉ | 906/1013 [2:09:15<23:30, 13.18s/it]
 90%|████████▉ | 907/1013 [2:09:22<20:16, 11.48s/it]
 90%|████████▉ | 908/1013 [2:09:30<18:00, 10.29s/it]
 90%|████████▉ | 909/1013 [2:09:37<16:08,  9.31s/it]
 90%|████████▉ | 910/1013 [2:09:44<14:52,  8.66s/it]
                                                    
{'loss': '9.889e-08', 'grad_norm': '2.205e-05', 'learning_rate': '1.587e-06', 'entropy': '1.284', 'num_tokens': '8.379e+06', 'logits/chosen': '2.341', 'logits/rejected': '-1.165', 'mean_token_accuracy': '0.661', 'rewards/chosen': '4.243', 'rewards/rejected': '-15.43', 'rewards/accuracies': '1', 'rewards/margins': '19.68', 'logps/chosen': '-65.78', 'logps/rejected': '-333.8', 'epoch': '2.693'}

 90%|████████▉ | 910/1013 [2:09:44<14:52,  8.66s/it]
 90%|████████▉ | 911/1013 [2:09:51<14:04,  8.28s/it]
 90%|█████████ | 912/1013 [2:09:59<13:26,  7.99s/it]
 90%|█████████ | 913/1013 [2:10:06<13:02,  7.83s/it]
 90%|█████████ | 914/1013 [2:10:13<12:29,  7.57s/it]
 90%|█████████ | 915/1013 [2:10:20<12:12,  7.48s/it]
 90%|█████████ | 916/1013 [2:10:28<11:56,  7.39s/it]
 91%|█████████ | 917/1013 [2:10:35<11:58,  7.48s/it]
 91%|█████████ | 918/1013 [2:10:43<11:52,  7.50s/it]
 91%|█████████ | 919/1013 [2:10:51<11:52,  7.59s/it]
 91%|█████████ | 920/1013 [2:10:58<11:45,  7.59s/it]
                                                    
{'loss': '5.674e-08', 'grad_norm': '2.933e-05', 'learning_rate': '1.299e-06', 'entropy': '1.282', 'num_tokens': '8.471e+06', 'logits/chosen': '2.308', 'logits/rejected': '-1.158', 'mean_token_accuracy': '0.6794', 'rewards/chosen': '4.262', 'rewards/rejected': '-15.31', 'rewards/accuracies': '1', 'rewards/margins': '19.57', 'logps/chosen': '-60.28', 'logps/rejected': '-329.9', 'epoch': '2.723'}

 91%|█████████ | 920/1013 [2:10:58<11:45,  7.59s/it]
 91%|█████████ | 921/1013 [2:11:05<11:24,  7.44s/it]
 91%|█████████ | 922/1013 [2:11:13<11:29,  7.58s/it]
 91%|█████████ | 923/1013 [2:11:20<11:08,  7.42s/it]
 91%|█████████ | 924/1013 [2:11:27<10:48,  7.28s/it]
 91%|█████████▏| 925/1013 [2:11:34<10:39,  7.26s/it]
 91%|█████████▏| 926/1013 [2:11:42<10:41,  7.37s/it]
 92%|█████████▏| 927/1013 [2:11:50<10:51,  7.57s/it]
 92%|█████████▏| 928/1013 [2:11:58<10:49,  7.64s/it]
 92%|█████████▏| 929/1013 [2:12:05<10:31,  7.52s/it]
 92%|█████████▏| 930/1013 [2:12:13<10:27,  7.56s/it]
                                                    
{'loss': '1.114e-07', 'grad_norm': '0.0001116', 'learning_rate': '1.039e-06', 'entropy': '1.262', 'num_tokens': '8.563e+06', 'logits/chosen': '2.324', 'logits/rejected': '-1.067', 'mean_token_accuracy': '0.6756', 'rewards/chosen': '4.329', 'rewards/rejected': '-15.12', 'rewards/accuracies': '1', 'rewards/margins': '19.45', 'logps/chosen': '-59.67', 'logps/rejected': '-327.1', 'epoch': '2.753'}

 92%|█████████▏| 930/1013 [2:12:13<10:27,  7.56s/it]
 92%|█████████▏| 931/1013 [2:12:21<10:26,  7.64s/it]
 92%|█████████▏| 932/1013 [2:12:27<09:57,  7.38s/it]
 92%|█████████▏| 933/1013 [2:12:35<10:00,  7.50s/it]
 92%|█████████▏| 934/1013 [2:12:43<09:58,  7.58s/it]
 92%|█████████▏| 935/1013 [2:12:50<09:48,  7.54s/it]
 92%|█████████▏| 936/1013 [2:12:58<09:32,  7.44s/it]
 92%|█████████▏| 937/1013 [2:13:04<09:12,  7.28s/it]
 93%|█████████▎| 938/1013 [2:13:12<09:15,  7.40s/it]
 93%|█████████▎| 939/1013 [2:13:20<09:17,  7.53s/it]
 93%|█████████▎| 940/1013 [2:13:27<09:01,  7.42s/it]
                                                    
{'loss': '1.418e-07', 'grad_norm': '0.000114', 'learning_rate': '8.078e-07', 'entropy': '1.295', 'num_tokens': '8.655e+06', 'logits/chosen': '2.221', 'logits/rejected': '-1.151', 'mean_token_accuracy': '0.6663', 'rewards/chosen': '4.4', 'rewards/rejected': '-15.25', 'rewards/accuracies': '1', 'rewards/margins': '19.65', 'logps/chosen': '-66.12', 'logps/rejected': '-330', 'epoch': '2.782'}

 93%|█████████▎| 940/1013 [2:13:27<09:01,  7.42s/it]
 93%|█████████▎| 941/1013 [2:13:35<08:54,  7.42s/it]
 93%|█████████▎| 942/1013 [2:13:41<08:35,  7.27s/it]
 93%|█████████▎| 943/1013 [2:13:49<08:38,  7.41s/it]
 93%|█████████▎| 944/1013 [2:13:57<08:29,  7.39s/it]
 93%|█████████▎| 945/1013 [2:14:04<08:28,  7.48s/it]
 93%|█████████▎| 946/1013 [2:14:12<08:19,  7.46s/it]
 93%|█████████▎| 947/1013 [2:14:19<08:00,  7.28s/it]
 94%|█████████▎| 948/1013 [2:14:26<08:02,  7.42s/it]
 94%|█████████▎| 949/1013 [2:14:33<07:46,  7.30s/it]
 94%|█████████▍| 950/1013 [2:14:41<07:41,  7.33s/it]
                                                    
{'loss': '2.465e-07', 'grad_norm': '0.0003052', 'learning_rate': '6.051e-07', 'entropy': '1.298', 'num_tokens': '8.746e+06', 'logits/chosen': '2.227', 'logits/rejected': '-1.133', 'mean_token_accuracy': '0.67', 'rewards/chosen': '4.108', 'rewards/rejected': '-14.89', 'rewards/accuracies': '1', 'rewards/margins': '19', 'logps/chosen': '-66.14', 'logps/rejected': '-324.2', 'epoch': '2.812'}

 94%|█████████▍| 950/1013 [2:14:41<07:41,  7.33s/it]
 94%|█████████▍| 951/1013 [2:14:48<07:41,  7.44s/it]
 94%|█████████▍| 952/1013 [2:14:56<07:32,  7.42s/it]
 94%|█████████▍| 953/1013 [2:15:04<07:32,  7.55s/it]
 94%|█████████▍| 954/1013 [2:15:11<07:18,  7.44s/it]
 94%|█████████▍| 955/1013 [2:15:19<07:18,  7.56s/it]
 94%|█████████▍| 956/1013 [2:15:26<07:12,  7.58s/it]
 94%|█████████▍| 957/1013 [2:15:34<07:01,  7.53s/it]
 95%|█████████▍| 958/1013 [2:15:42<07:00,  7.64s/it]
 95%|█████████▍| 959/1013 [2:15:49<06:50,  7.60s/it]
 95%|█████████▍| 960/1013 [2:15:56<06:36,  7.48s/it]
                                                    
{'loss': '1.341e-07', 'grad_norm': '2.444e-06', 'learning_rate': '4.313e-07', 'entropy': '1.264', 'num_tokens': '8.841e+06', 'logits/chosen': '2.29', 'logits/rejected': '-1.155', 'mean_token_accuracy': '0.6711', 'rewards/chosen': '4.389', 'rewards/rejected': '-15.3', 'rewards/accuracies': '1', 'rewards/margins': '19.69', 'logps/chosen': '-62.26', 'logps/rejected': '-332.4', 'epoch': '2.841'}

 95%|█████████▍| 960/1013 [2:15:56<06:36,  7.48s/it]
 95%|█████████▍| 961/1013 [2:16:03<06:20,  7.31s/it]
 95%|█████████▍| 962/1013 [2:16:11<06:15,  7.35s/it]
 95%|█████████▌| 963/1013 [2:16:18<06:07,  7.35s/it]
 95%|█████████▌| 964/1013 [2:16:25<06:02,  7.39s/it]
 95%|█████████▌| 965/1013 [2:16:33<05:52,  7.35s/it]
 95%|█████████▌| 966/1013 [2:16:40<05:47,  7.39s/it]
 95%|█████████▌| 967/1013 [2:16:48<05:45,  7.52s/it]
 96%|█████████▌| 968/1013 [2:16:55<05:34,  7.44s/it]
 96%|█████████▌| 969/1013 [2:17:03<05:25,  7.39s/it]
 96%|█████████▌| 970/1013 [2:17:10<05:16,  7.35s/it]
                                                    
{'loss': '1.441e-07', 'grad_norm': '1.836e-05', 'learning_rate': '2.866e-07', 'entropy': '1.286', 'num_tokens': '8.932e+06', 'logits/chosen': '2.317', 'logits/rejected': '-1.187', 'mean_token_accuracy': '0.669', 'rewards/chosen': '4.227', 'rewards/rejected': '-15.06', 'rewards/accuracies': '1', 'rewards/margins': '19.29', 'logps/chosen': '-62.73', 'logps/rejected': '-326.8', 'epoch': '2.871'}

 96%|█████████▌| 970/1013 [2:17:10<05:16,  7.35s/it]
 96%|█████████▌| 971/1013 [2:17:18<05:13,  7.47s/it]
 96%|█████████▌| 972/1013 [2:17:25<05:07,  7.49s/it]
 96%|█████████▌| 973/1013 [2:17:33<05:03,  7.59s/it]
 96%|█████████▌| 974/1013 [2:17:40<04:51,  7.48s/it]
 96%|█████████▌| 975/1013 [2:17:47<04:41,  7.40s/it]
 96%|█████████▋| 976/1013 [2:17:55<04:40,  7.57s/it]
 96%|█████████▋| 977/1013 [2:18:03<04:35,  7.64s/it]
 97%|█████████▋| 978/1013 [2:18:11<04:28,  7.68s/it]
 97%|█████████▋| 979/1013 [2:18:18<04:18,  7.59s/it]
 97%|█████████▋| 980/1013 [2:18:26<04:11,  7.62s/it]
                                                    
{'loss': '1.181e-07', 'grad_norm': '4.888e-06', 'learning_rate': '1.713e-07', 'entropy': '1.278', 'num_tokens': '9.029e+06', 'logits/chosen': '2.352', 'logits/rejected': '-1.175', 'mean_token_accuracy': '0.6487', 'rewards/chosen': '4.309', 'rewards/rejected': '-15.18', 'rewards/accuracies': '1', 'rewards/margins': '19.49', 'logps/chosen': '-69.6', 'logps/rejected': '-329.8', 'epoch': '2.901'}

 97%|█████████▋| 980/1013 [2:18:26<04:11,  7.62s/it]
 97%|█████████▋| 981/1013 [2:18:33<03:55,  7.37s/it]
 97%|█████████▋| 982/1013 [2:18:40<03:49,  7.42s/it]
 97%|█████████▋| 983/1013 [2:18:47<03:39,  7.32s/it]
 97%|█████████▋| 984/1013 [2:18:55<03:31,  7.30s/it]
 97%|█████████▋| 985/1013 [2:19:03<03:30,  7.51s/it]
 97%|█████████▋| 986/1013 [2:19:10<03:25,  7.60s/it]
 97%|█████████▋| 987/1013 [2:19:18<03:16,  7.58s/it]
 98%|█████████▊| 988/1013 [2:19:26<03:10,  7.60s/it]
 98%|█████████▊| 989/1013 [2:19:33<02:59,  7.50s/it]
 98%|█████████▊| 990/1013 [2:19:40<02:51,  7.47s/it]
                                                    
{'loss': '1.604e-07', 'grad_norm': '7.063e-06', 'learning_rate': '8.539e-08', 'entropy': '1.266', 'num_tokens': '9.121e+06', 'logits/chosen': '2.306', 'logits/rejected': '-1.07', 'mean_token_accuracy': '0.6714', 'rewards/chosen': '4.18', 'rewards/rejected': '-15.49', 'rewards/accuracies': '1', 'rewards/margins': '19.67', 'logps/chosen': '-65.15', 'logps/rejected': '-333.3', 'epoch': '2.93'}

 98%|█████████▊| 990/1013 [2:19:40<02:51,  7.47s/it]
 98%|█████████▊| 991/1013 [2:19:47<02:42,  7.38s/it]
 98%|█████████▊| 992/1013 [2:19:55<02:34,  7.36s/it]
 98%|█████████▊| 993/1013 [2:20:03<02:29,  7.49s/it]
 98%|█████████▊| 994/1013 [2:20:10<02:20,  7.39s/it]
 98%|█████████▊| 995/1013 [2:20:17<02:14,  7.45s/it]
 98%|█████████▊| 996/1013 [2:20:25<02:09,  7.64s/it]
 98%|█████████▊| 997/1013 [2:20:33<02:01,  7.60s/it]
 99%|█████████▊| 998/1013 [2:20:40<01:53,  7.54s/it]
 99%|█████████▊| 999/1013 [2:20:48<01:46,  7.62s/it]
 99%|█████████▊| 1000/1013 [2:20:55<01:36,  7.44s/it]
                                                     
{'loss': '1.205e-07', 'grad_norm': '3.263e-06', 'learning_rate': '2.907e-08', 'entropy': '1.28', 'num_tokens': '9.214e+06', 'logits/chosen': '2.326', 'logits/rejected': '-1.083', 'mean_token_accuracy': '0.6543', 'rewards/chosen': '4.286', 'rewards/rejected': '-15.53', 'rewards/accuracies': '1', 'rewards/margins': '19.81', 'logps/chosen': '-65.36', 'logps/rejected': '-333.5', 'epoch': '2.96'}

 99%|█████████▊| 1000/1013 [2:20:55<01:36,  7.44s/it]

  0%|          | 0/143 [00:00<?, ?it/s]

  1%|▏         | 2/143 [00:00<00:22,  6.31it/s]

  2%|▏         | 3/143 [00:00<00:30,  4.62it/s]

  3%|▎         | 4/143 [00:00<00:32,  4.32it/s]

  3%|▎         | 5/143 [00:01<00:34,  3.95it/s]

  4%|▍         | 6/143 [00:01<00:33,  4.05it/s]

  5%|▍         | 7/143 [00:01<00:31,  4.28it/s]

  6%|▌         | 8/143 [00:01<00:33,  4.02it/s]

  6%|▋         | 9/143 [00:02<00:31,  4.28it/s]

  7%|▋         | 10/143 [00:02<00:32,  4.14it/s]

  8%|▊         | 11/143 [00:02<00:31,  4.18it/s]

  8%|▊         | 12/143 [00:02<00:33,  3.89it/s]

  9%|▉         | 13/143 [00:03<00:34,  3.82it/s]

 10%|▉         | 14/143 [00:03<00:35,  3.61it/s]

 10%|█         | 15/143 [00:03<00:35,  3.63it/s]

 11%|█         | 16/143 [00:03<00:32,  3.93it/s]

 12%|█▏        | 17/143 [00:04<00:29,  4.20it/s]

 13%|█▎        | 18/143 [00:04<00:29,  4.27it/s]

 13%|█▎        | 19/143 [00:04<00:32,  3.80it/s]

 14%|█▍        | 20/143 [00:05<00:34,  3.61it/s]

 15%|█▍        | 21/143 [00:05<00:33,  3.69it/s]

 15%|█▌        | 22/143 [00:05<00:33,  3.65it/s]

 16%|█▌        | 23/143 [00:05<00:32,  3.73it/s]

 17%|█▋        | 24/143 [00:06<00:32,  3.67it/s]

 17%|█▋        | 25/143 [00:06<00:31,  3.81it/s]

 18%|█▊        | 26/143 [00:06<00:29,  3.97it/s]

 19%|█▉        | 27/143 [00:06<00:32,  3.53it/s]

 20%|█▉        | 28/143 [00:07<00:33,  3.44it/s]

 20%|██        | 29/143 [00:07<00:33,  3.45it/s]

 21%|██        | 30/143 [00:07<00:33,  3.38it/s]

 22%|██▏       | 31/143 [00:08<00:32,  3.43it/s]

 22%|██▏       | 32/143 [00:08<00:30,  3.64it/s]

 23%|██▎       | 33/143 [00:08<00:28,  3.85it/s]

 24%|██▍       | 34/143 [00:08<00:28,  3.88it/s]

 24%|██▍       | 35/143 [00:09<00:28,  3.76it/s]

 25%|██▌       | 36/143 [00:09<00:27,  3.94it/s]

 26%|██▌       | 37/143 [00:09<00:26,  3.94it/s]

 27%|██▋       | 38/143 [00:09<00:26,  3.93it/s]

 27%|██▋       | 39/143 [00:10<00:28,  3.69it/s]

 28%|██▊       | 40/143 [00:10<00:26,  3.88it/s]

 29%|██▊       | 41/143 [00:10<00:36,  2.81it/s]

 29%|██▉       | 42/143 [00:11<00:34,  2.93it/s]

 30%|███       | 43/143 [00:11<00:32,  3.07it/s]

 31%|███       | 44/143 [00:11<00:30,  3.28it/s]

 31%|███▏      | 45/143 [00:12<00:27,  3.52it/s]

 32%|███▏      | 46/143 [00:12<00:32,  2.94it/s]

 33%|███▎      | 47/143 [00:12<00:31,  3.06it/s]

 34%|███▎      | 48/143 [00:13<00:29,  3.27it/s]

 34%|███▍      | 49/143 [00:13<00:28,  3.33it/s]

 35%|███▍      | 50/143 [00:13<00:27,  3.39it/s]

 36%|███▌      | 51/143 [00:13<00:26,  3.43it/s]

 36%|███▋      | 52/143 [00:14<00:25,  3.55it/s]

 37%|███▋      | 53/143 [00:14<00:24,  3.72it/s]

 38%|███▊      | 54/143 [00:14<00:25,  3.49it/s]

 38%|███▊      | 55/143 [00:14<00:24,  3.60it/s]

 39%|███▉      | 56/143 [00:15<00:22,  3.79it/s]

 40%|███▉      | 57/143 [00:15<00:22,  3.82it/s]

 41%|████      | 58/143 [00:15<00:20,  4.09it/s]

 41%|████▏     | 59/143 [00:15<00:20,  4.19it/s]

 42%|████▏     | 60/143 [00:16<00:20,  3.97it/s]

 43%|████▎     | 61/143 [00:16<00:19,  4.21it/s]

 43%|████▎     | 62/143 [00:16<00:22,  3.59it/s]

 44%|████▍     | 63/143 [00:17<00:22,  3.61it/s]

 45%|████▍     | 64/143 [00:17<00:23,  3.41it/s]

 45%|████▌     | 65/143 [00:17<00:22,  3.41it/s]

 46%|████▌     | 66/143 [00:18<00:24,  3.13it/s]

 47%|████▋     | 67/143 [00:18<00:22,  3.32it/s]

 48%|████▊     | 68/143 [00:18<00:22,  3.40it/s]

 48%|████▊     | 69/143 [00:18<00:24,  3.08it/s]

 49%|████▉     | 70/143 [00:19<00:21,  3.39it/s]

 50%|████▉     | 71/143 [00:19<00:20,  3.43it/s]

 50%|█████     | 72/143 [00:19<00:20,  3.46it/s]

 51%|█████     | 73/143 [00:20<00:20,  3.42it/s]

 52%|█████▏    | 74/143 [00:20<00:20,  3.40it/s]

 52%|█████▏    | 75/143 [00:20<00:20,  3.35it/s]

 53%|█████▎    | 76/143 [00:20<00:19,  3.49it/s]

 54%|█████▍    | 77/143 [00:21<00:18,  3.62it/s]

 55%|█████▍    | 78/143 [00:21<00:16,  3.92it/s]

 55%|█████▌    | 79/143 [00:21<00:16,  3.92it/s]

 56%|█████▌    | 80/143 [00:22<00:18,  3.36it/s]

 57%|█████▋    | 81/143 [00:22<00:16,  3.69it/s]

 57%|█████▋    | 82/143 [00:22<00:18,  3.37it/s]

 58%|█████▊    | 83/143 [00:23<00:22,  2.65it/s]

 59%|█████▊    | 84/143 [00:23<00:20,  2.82it/s]

 59%|█████▉    | 85/143 [00:23<00:18,  3.08it/s]

 60%|██████    | 86/143 [00:23<00:15,  3.61it/s]

 61%|██████    | 87/143 [00:24<00:14,  3.78it/s]

 62%|██████▏   | 88/143 [00:24<00:14,  3.74it/s]

 62%|██████▏   | 89/143 [00:24<00:14,  3.68it/s]

 63%|██████▎   | 90/143 [00:24<00:14,  3.75it/s]

 64%|██████▎   | 91/143 [00:25<00:12,  4.04it/s]

 64%|██████▍   | 92/143 [00:25<00:13,  3.90it/s]

 65%|██████▌   | 93/143 [00:25<00:12,  3.91it/s]

 66%|██████▌   | 94/143 [00:26<00:13,  3.65it/s]

 66%|██████▋   | 95/143 [00:26<00:14,  3.28it/s]

 67%|██████▋   | 96/143 [00:26<00:13,  3.51it/s]

 68%|██████▊   | 97/143 [00:26<00:11,  3.85it/s]

 69%|██████▊   | 98/143 [00:27<00:11,  3.97it/s]

 69%|██████▉   | 99/143 [00:27<00:11,  3.77it/s]

 70%|██████▉   | 100/143 [00:27<00:12,  3.50it/s]

 71%|███████   | 101/143 [00:27<00:11,  3.74it/s]

 71%|███████▏  | 102/143 [00:28<00:11,  3.69it/s]

 72%|███████▏  | 103/143 [00:28<00:11,  3.52it/s]

 73%|███████▎  | 104/143 [00:28<00:10,  3.77it/s]

 73%|███████▎  | 105/143 [00:29<00:10,  3.68it/s]

 74%|███████▍  | 106/143 [00:29<00:10,  3.63it/s]

 75%|███████▍  | 107/143 [00:29<00:09,  3.79it/s]

 76%|███████▌  | 108/143 [00:29<00:08,  3.92it/s]

 76%|███████▌  | 109/143 [00:30<00:08,  3.91it/s]

 77%|███████▋  | 110/143 [00:30<00:08,  4.01it/s]

 78%|███████▊  | 111/143 [00:30<00:08,  3.65it/s]

 78%|███████▊  | 112/143 [00:30<00:08,  3.80it/s]

 79%|███████▉  | 113/143 [00:31<00:09,  3.32it/s]

 80%|███████▉  | 114/143 [00:31<00:08,  3.49it/s]

 80%|████████  | 115/143 [00:31<00:08,  3.49it/s]

 81%|████████  | 116/143 [00:31<00:07,  3.68it/s]

 82%|████████▏ | 117/143 [00:32<00:07,  3.52it/s]

 83%|████████▎ | 118/143 [00:32<00:06,  3.85it/s]

 83%|████████▎ | 119/143 [00:32<00:05,  4.14it/s]

 84%|████████▍ | 120/143 [00:32<00:05,  3.93it/s]

 85%|████████▍ | 121/143 [00:33<00:05,  3.75it/s]

 85%|████████▌ | 122/143 [00:33<00:05,  4.17it/s]

 86%|████████▌ | 123/143 [00:33<00:05,  3.90it/s]

 87%|████████▋ | 124/143 [00:33<00:04,  4.00it/s]

 87%|████████▋ | 125/143 [00:34<00:05,  3.35it/s]

 88%|████████▊ | 126/143 [00:34<00:04,  3.58it/s]

 89%|████████▉ | 127/143 [00:34<00:04,  3.55it/s]

 90%|████████▉ | 128/143 [00:35<00:04,  3.50it/s]

 90%|█████████ | 129/143 [00:35<00:03,  3.69it/s]

 91%|█████████ | 130/143 [00:35<00:03,  3.88it/s]

 92%|█████████▏| 131/143 [00:35<00:02,  4.03it/s]

 92%|█████████▏| 132/143 [00:36<00:02,  4.00it/s]

 93%|█████████▎| 133/143 [00:36<00:02,  3.52it/s]

 94%|█████████▎| 134/143 [00:36<00:02,  3.51it/s]

 94%|█████████▍| 135/143 [00:37<00:02,  3.40it/s]

 95%|█████████▌| 136/143 [00:37<00:01,  3.62it/s]

 96%|█████████▌| 137/143 [00:37<00:01,  3.47it/s]

 97%|█████████▋| 138/143 [00:37<00:01,  3.46it/s]

 97%|█████████▋| 139/143 [00:38<00:01,  3.59it/s]

 98%|█████████▊| 140/143 [00:38<00:00,  3.90it/s]

 99%|█████████▊| 141/143 [00:38<00:00,  3.82it/s]

 99%|█████████▉| 142/143 [00:38<00:00,  3.73it/s]

100%|██████████| 143/143 [00:39<00:00,  3.56it/s]
                                                     

                                                 
{'eval_loss': '6.491e-06', 'eval_runtime': '39.71', 'eval_samples_per_second': '7.177', 'eval_steps_per_second': '3.601', 'eval_entropy': '1.28', 'eval_num_tokens': '9.214e+06', 'eval_logits/chosen': '2.244', 'eval_logits/rejected': '-1.159', 'eval_mean_token_accuracy': '0.6638', 'eval_rewards/chosen': '4.315', 'eval_rewards/rejected': '-15.16', 'eval_rewards/accuracies': '1', 'eval_rewards/margins': '19.48', 'eval_logps/chosen': '-68', 'eval_logps/rejected': '-326.7', 'epoch': '2.96'}

 99%|█████████▊| 1000/1013 [2:21:35<01:36,  7.44s/it]

100%|██████████| 143/143 [00:39<00:00,  3.56it/s]

                                                 

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:32<00:00, 32.22s/it]
Writing model shards: 100%|██████████| 1/1 [00:32<00:00, 32.22s/it]

 99%|█████████▉| 1001/1013 [2:22:56<08:16, 41.38s/it]
 99%|█████████▉| 1002/1013 [2:23:03<05:43, 31.20s/it]
 99%|█████████▉| 1003/1013 [2:23:11<04:01, 24.10s/it]
 99%|█████████▉| 1004/1013 [2:23:18<02:51, 19.06s/it]
 99%|█████████▉| 1005/1013 [2:23:26<02:05, 15.69s/it]
 99%|█████████▉| 1006/1013 [2:23:33<01:32, 13.20s/it]
 99%|█████████▉| 1007/1013 [2:23:41<01:08, 11.47s/it]
100%|█████████▉| 1008/1013 [2:23:48<00:51, 10.22s/it]
100%|█████████▉| 1009/1013 [2:23:55<00:37,  9.31s/it]
100%|█████████▉| 1010/1013 [2:24:03<00:26,  8.86s/it]
                                                     
{'loss': '1.837e-07', 'grad_norm': '2.098e-05', 'learning_rate': '2.373e-09', 'entropy': '1.275', 'num_tokens': '9.308e+06', 'logits/chosen': '2.248', 'logits/rejected': '-1.168', 'mean_token_accuracy': '0.6662', 'rewards/chosen': '4.232', 'rewards/rejected': '-15.46', 'rewards/accuracies': '1', 'rewards/margins': '19.69', 'logps/chosen': '-62.23', 'logps/rejected': '-334.2', 'epoch': '2.99'}

100%|█████████▉| 1010/1013 [2:24:03<00:26,  8.86s/it]
100%|█████████▉| 1011/1013 [2:24:11<00:17,  8.53s/it]
100%|█████████▉| 1012/1013 [2:24:18<00:08,  8.17s/it]
100%|██████████| 1013/1013 [2:24:25<00:00,  7.96s/it]

  0%|          | 0/143 [00:00<?, ?it/s]

  1%|▏         | 2/143 [00:00<00:22,  6.27it/s]

  2%|▏         | 3/143 [00:00<00:30,  4.62it/s]

  3%|▎         | 4/143 [00:00<00:32,  4.34it/s]

  3%|▎         | 5/143 [00:01<00:34,  3.96it/s]

  4%|▍         | 6/143 [00:01<00:33,  4.04it/s]

  5%|▍         | 7/143 [00:01<00:31,  4.26it/s]

  6%|▌         | 8/143 [00:01<00:33,  3.99it/s]

  6%|▋         | 9/143 [00:02<00:31,  4.26it/s]

  7%|▋         | 10/143 [00:02<00:32,  4.14it/s]

  8%|▊         | 11/143 [00:02<00:31,  4.19it/s]

  8%|▊         | 12/143 [00:02<00:33,  3.90it/s]

  9%|▉         | 13/143 [00:03<00:34,  3.82it/s]

 10%|▉         | 14/143 [00:03<00:35,  3.62it/s]

 10%|█         | 15/143 [00:03<00:35,  3.63it/s]

 11%|█         | 16/143 [00:03<00:32,  3.94it/s]

 12%|█▏        | 17/143 [00:04<00:29,  4.21it/s]

 13%|█▎        | 18/143 [00:04<00:29,  4.29it/s]

 13%|█▎        | 19/143 [00:04<00:32,  3.83it/s]

 14%|█▍        | 20/143 [00:04<00:33,  3.63it/s]

 15%|█▍        | 21/143 [00:05<00:32,  3.71it/s]

 15%|█▌        | 22/143 [00:05<00:33,  3.66it/s]

 16%|█▌        | 23/143 [00:05<00:32,  3.74it/s]

 17%|█▋        | 24/143 [00:06<00:32,  3.68it/s]

 17%|█▋        | 25/143 [00:06<00:30,  3.82it/s]

 18%|█▊        | 26/143 [00:06<00:29,  3.99it/s]

 19%|█▉        | 27/143 [00:06<00:32,  3.54it/s]

 20%|█▉        | 28/143 [00:07<00:33,  3.44it/s]

 20%|██        | 29/143 [00:07<00:32,  3.46it/s]

 21%|██        | 30/143 [00:07<00:33,  3.39it/s]

 22%|██▏       | 31/143 [00:08<00:32,  3.44it/s]

 22%|██▏       | 32/143 [00:08<00:30,  3.64it/s]

 23%|██▎       | 33/143 [00:08<00:28,  3.85it/s]

 24%|██▍       | 34/143 [00:08<00:28,  3.89it/s]

 24%|██▍       | 35/143 [00:09<00:28,  3.76it/s]

 25%|██▌       | 36/143 [00:09<00:27,  3.95it/s]

 26%|██▌       | 37/143 [00:09<00:26,  3.95it/s]

 27%|██▋       | 38/143 [00:10<00:36,  2.84it/s]

 27%|██▋       | 39/143 [00:10<00:35,  2.95it/s]

 28%|██▊       | 40/143 [00:10<00:31,  3.28it/s]

 29%|██▊       | 41/143 [00:10<00:29,  3.44it/s]

 29%|██▉       | 42/143 [00:11<00:29,  3.38it/s]

 30%|███       | 43/143 [00:11<00:29,  3.41it/s]

 31%|███       | 44/143 [00:11<00:27,  3.54it/s]

 31%|███▏      | 45/143 [00:12<00:26,  3.72it/s]

 32%|███▏      | 46/143 [00:12<00:31,  3.04it/s]

 33%|███▎      | 47/143 [00:12<00:30,  3.13it/s]

 34%|███▎      | 48/143 [00:13<00:28,  3.33it/s]

 34%|███▍      | 49/143 [00:13<00:27,  3.39it/s]

 35%|███▍      | 50/143 [00:13<00:27,  3.44it/s]

 36%|███▌      | 51/143 [00:13<00:26,  3.47it/s]

 36%|███▋      | 52/143 [00:14<00:25,  3.59it/s]

 37%|███▋      | 53/143 [00:14<00:24,  3.74it/s]

 38%|███▊      | 54/143 [00:14<00:25,  3.50it/s]

 38%|███▊      | 55/143 [00:14<00:24,  3.61it/s]

 39%|███▉      | 56/143 [00:15<00:22,  3.79it/s]

 40%|███▉      | 57/143 [00:15<00:22,  3.83it/s]

 41%|████      | 58/143 [00:15<00:20,  4.10it/s]

 41%|████▏     | 59/143 [00:15<00:19,  4.20it/s]

 42%|████▏     | 60/143 [00:16<00:20,  3.99it/s]

 43%|████▎     | 61/143 [00:16<00:19,  4.23it/s]

 43%|████▎     | 62/143 [00:16<00:22,  3.60it/s]

 44%|████▍     | 63/143 [00:17<00:22,  3.62it/s]

 45%|████▍     | 64/143 [00:17<00:23,  3.42it/s]

 45%|████▌     | 65/143 [00:17<00:22,  3.41it/s]

 46%|████▌     | 66/143 [00:18<00:24,  3.13it/s]

 47%|████▋     | 67/143 [00:18<00:22,  3.32it/s]

 48%|████▊     | 68/143 [00:18<00:21,  3.41it/s]

 48%|████▊     | 69/143 [00:18<00:23,  3.09it/s]

 49%|████▉     | 70/143 [00:19<00:21,  3.38it/s]

 50%|████▉     | 71/143 [00:19<00:20,  3.43it/s]

 50%|█████     | 72/143 [00:19<00:20,  3.46it/s]

 51%|█████     | 73/143 [00:20<00:20,  3.42it/s]

 52%|█████▏    | 74/143 [00:20<00:20,  3.41it/s]

 52%|█████▏    | 75/143 [00:20<00:20,  3.36it/s]

 53%|█████▎    | 76/143 [00:21<00:24,  2.72it/s]

 54%|█████▍    | 77/143 [00:21<00:21,  3.01it/s]

 55%|█████▍    | 78/143 [00:21<00:19,  3.39it/s]

 55%|█████▌    | 79/143 [00:21<00:18,  3.54it/s]

 56%|█████▌    | 80/143 [00:22<00:19,  3.17it/s]

 57%|█████▋    | 81/143 [00:22<00:17,  3.53it/s]

 57%|█████▋    | 82/143 [00:22<00:18,  3.28it/s]

 58%|█████▊    | 83/143 [00:23<00:18,  3.20it/s]

 59%|█████▊    | 84/143 [00:23<00:18,  3.24it/s]

 59%|█████▉    | 85/143 [00:23<00:17,  3.41it/s]

 60%|██████    | 86/143 [00:23<00:14,  3.93it/s]

 61%|██████    | 87/143 [00:24<00:13,  4.02it/s]

 62%|██████▏   | 88/143 [00:24<00:14,  3.90it/s]

 62%|██████▏   | 89/143 [00:24<00:14,  3.78it/s]

 63%|██████▎   | 90/143 [00:24<00:13,  3.83it/s]

 64%|██████▎   | 91/143 [00:25<00:12,  4.11it/s]

 64%|██████▍   | 92/143 [00:25<00:12,  3.95it/s]

 65%|██████▌   | 93/143 [00:25<00:12,  3.95it/s]

 66%|██████▌   | 94/143 [00:25<00:13,  3.67it/s]

 66%|██████▋   | 95/143 [00:26<00:14,  3.29it/s]

 67%|██████▋   | 96/143 [00:26<00:13,  3.52it/s]

 68%|██████▊   | 97/143 [00:26<00:11,  3.86it/s]

 69%|██████▊   | 98/143 [00:27<00:11,  3.98it/s]

 69%|██████▉   | 99/143 [00:27<00:11,  3.77it/s]

 70%|██████▉   | 100/143 [00:27<00:12,  3.51it/s]

 71%|███████   | 101/143 [00:27<00:11,  3.75it/s]

 71%|███████▏  | 102/143 [00:28<00:11,  3.69it/s]

 72%|███████▏  | 103/143 [00:28<00:11,  3.52it/s]

 73%|███████▎  | 104/143 [00:28<00:10,  3.77it/s]

 73%|███████▎  | 105/143 [00:28<00:10,  3.69it/s]

 74%|███████▍  | 106/143 [00:29<00:10,  3.65it/s]

 75%|███████▍  | 107/143 [00:29<00:09,  3.80it/s]

 76%|███████▌  | 108/143 [00:29<00:08,  3.93it/s]

 76%|███████▌  | 109/143 [00:29<00:08,  3.93it/s]

 77%|███████▋  | 110/143 [00:30<00:08,  4.03it/s]

 78%|███████▊  | 111/143 [00:30<00:08,  3.66it/s]

 78%|███████▊  | 112/143 [00:30<00:08,  3.82it/s]

 79%|███████▉  | 113/143 [00:31<00:09,  3.32it/s]

 80%|███████▉  | 114/143 [00:31<00:08,  3.48it/s]

 80%|████████  | 115/143 [00:31<00:08,  3.48it/s]

 81%|████████  | 116/143 [00:31<00:07,  3.69it/s]

 82%|████████▏ | 117/143 [00:32<00:09,  2.81it/s]

 83%|████████▎ | 118/143 [00:32<00:07,  3.23it/s]

 83%|████████▎ | 119/143 [00:32<00:06,  3.61it/s]

 84%|████████▍ | 120/143 [00:33<00:06,  3.58it/s]

 85%|████████▍ | 121/143 [00:33<00:06,  3.52it/s]

 85%|████████▌ | 122/143 [00:33<00:05,  3.97it/s]

 86%|████████▌ | 123/143 [00:33<00:05,  3.78it/s]

 87%|████████▋ | 124/143 [00:34<00:04,  3.91it/s]

 87%|████████▋ | 125/143 [00:34<00:04,  4.34it/s]

 88%|████████▊ | 126/143 [00:34<00:03,  4.31it/s]

 89%|████████▉ | 127/143 [00:34<00:03,  4.02it/s]

 90%|████████▉ | 128/143 [00:35<00:03,  3.82it/s]

 90%|█████████ | 129/143 [00:35<00:03,  3.92it/s]

 91%|█████████ | 130/143 [00:35<00:03,  4.06it/s]

 92%|█████████▏| 131/143 [00:35<00:02,  4.17it/s]

 92%|█████████▏| 132/143 [00:36<00:02,  4.09it/s]

 93%|█████████▎| 133/143 [00:36<00:02,  3.56it/s]

 94%|█████████▎| 134/143 [00:36<00:02,  3.54it/s]

 94%|█████████▍| 135/143 [00:37<00:02,  3.42it/s]

 95%|█████████▌| 136/143 [00:37<00:01,  3.63it/s]

 96%|█████████▌| 137/143 [00:37<00:01,  3.49it/s]

 97%|█████████▋| 138/143 [00:37<00:01,  3.48it/s]

 97%|█████████▋| 139/143 [00:38<00:01,  3.60it/s]

 98%|█████████▊| 140/143 [00:38<00:00,  3.90it/s]

 99%|█████████▊| 141/143 [00:38<00:00,  3.81it/s]

 99%|█████████▉| 142/143 [00:38<00:00,  3.70it/s]

100%|██████████| 143/143 [00:39<00:00,  3.55it/s]
                                                     

                                                 
{'eval_loss': '6.491e-06', 'eval_runtime': '39.69', 'eval_samples_per_second': '7.181', 'eval_steps_per_second': '3.603', 'eval_entropy': '1.28', 'eval_num_tokens': '9.336e+06', 'eval_logits/chosen': '2.244', 'eval_logits/rejected': '-1.159', 'eval_mean_token_accuracy': '0.6638', 'eval_rewards/chosen': '4.315', 'eval_rewards/rejected': '-15.16', 'eval_rewards/accuracies': '1', 'eval_rewards/margins': '19.48', 'eval_logps/chosen': '-68', 'eval_logps/rejected': '-326.7', 'epoch': '2.999'}

100%|██████████| 1013/1013 [2:25:05<00:00,  7.96s/it]

100%|██████████| 143/143 [00:39<00:00,  3.55it/s]

                                                 

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]

Writing model shards: 100%|██████████| 1/1 [00:34<00:00, 34.93s/it]
Writing model shards: 100%|██████████| 1/1 [00:34<00:00, 34.93s/it]

                                                     
{'train_runtime': '8781', 'train_samples_per_second': '1.846', 'train_steps_per_second': '0.115', 'train_loss': '0.005771', 'epoch': '2.999'}

100%|██████████| 1013/1013 [2:26:21<00:00,  7.96s/it]
100%|██████████| 1013/1013 [2:26:21<00:00,  8.67s/it]
[2026-04-16 13:05:08,498] [INFO] [axolotl.train.save_trained_model:267] [PID:135] Training completed! Saving trained model to kikiyaa/qwen-dpo-finetuned-ver2.

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]
Writing model shards: 100%|██████████| 1/1 [00:34<00:00, 34.34s/it]
Writing model shards: 100%|██████████| 1/1 [00:34<00:00, 34.34s/it]
[2026-04-16 13:05:42,881] [INFO] [axolotl.train.save_trained_model:386] [PID:135] Model successfully saved to kikiyaa/qwen-dpo-finetuned-ver2