[2026-04-24 20:51:03,702] [DEBUG] [axolotl.utils.config.resolve_dtype:74] [PID:293513] bf16 support detected, enabling for this configuration. [2026-04-24 20:51:03,717] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:293513] baseline 0.000GB () [2026-04-24 20:51:03,718] [INFO] [axolotl.cli.config.load_cfg:341] [PID:293513] config: { "activation_offloading": true, "adam_beta1": 0.9, "adam_beta2": 0.95, "axolotl_config_path": "/e/scratch/jureap59/feuer1/code/axolotl_configs/qwen3_8b_sera_v4_1000_v6.yaml", "base_model": "Qwen/Qwen3-8B", "base_model_config": "Qwen/Qwen3-8B", "batch_size": 32, "bf16": true, "capabilities": { "bf16": true, "compute_capability": "sm_90", "fp8": true, "n_gpu": 4, "n_node": 1, "tf32": true }, "chat_template": "tokenizer_default", "context_parallel_size": 1, "dataloader_num_workers": 4, "dataloader_pin_memory": true, "dataloader_prefetch_factor": 256, "dataset_num_proc": 288, "dataset_prepared_path": "/e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/sera-v4-1000-v6", "datasets": [ { "chat_template": "tokenizer_default", "ds_type": "json", "field_messages": "messages", "message_field_training": "train", "message_property_mappings": { "content": "content", "role": "role" }, "path": "/e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl", "trust_remote_code": false, "type": "chat_template" } ], "ddp": true, "deepspeed": { "bf16": { "enabled": true }, "gradient_accumulation_steps": "auto", "gradient_clipping": "auto", "train_batch_size": "auto", "train_micro_batch_size_per_gpu": "auto", "wall_clock_breakdown": false, "zero_optimization": { "contiguous_gradients": true, "gather_16bit_weights_on_model_save": true, "max_live_parameters": 0, "max_reuse_distance": 0, "overlap_comm": true, "reduce_bucket_size": "auto", "stage": 3, "stage3_param_persistence_threshold": "auto", "stage3_prefetch_bucket_size": "auto", "sub_group_size": 0 } }, "device": "cuda:0", "device_map": { "": 0 }, "dion_rank_fraction": 1.0, "dion_rank_multiple_of": 1, "eaft_alpha": 1.0, "eaft_k": 20, "env_capabilities": { "torch_version": "2.9.1" }, "eval_batch_size": 1, "eval_causal_lm_metrics": [ "sacrebleu", "comet", "ter", "chrf" ], "eval_max_new_tokens": 128, "eval_table_size": 0, "evals_per_epoch": 0, "experimental_skip_move_to_device": true, "flash_attention": true, "fp16": false, "generate_samples": false, "generation_do_sample": true, "generation_max_new_tokens": 50, "generation_prompt_ratio": 0.5, "generation_temperature": 0.7, "gradient_accumulation_steps": 8, "gradient_checkpointing": true, "gradient_checkpointing_kwargs": { "use_reentrant": true }, "include_tkps": true, "layer_offloading": false, "learning_rate": 1e-05, "lisa_layers_attribute": "model.layers", "load_best_model_at_end": false, "load_in_4bit": false, "load_in_8bit": false, "local_rank": 0, "logging_steps": 1, "lora_dropout": 0.0, "loraplus_lr_embedding": 1e-06, "loss_watchdog_patience": 3, "loss_watchdog_threshold": 5.0, "lr_scheduler": "cosine", "max_grad_norm": 1.0, "mean_resizing_embeddings": false, "merge_method": "memory_efficient", "micro_batch_size": 1, "model_config_type": "qwen3", "num_epochs": 6.0, "num_generation_samples": 3, "optimizer": "adamw_torch", "otel_metrics_host": "localhost", "otel_metrics_port": 8000, "output_dir": "/e/data1/datasets/playground/ot-baf/checkpoints/sera-v4-1000-axolotl__Qwen3-8B-v6", "pretrain_multipack_attn": true, "profiler_steps_start": 0, "qlora_sharded_model_loading": false, "quantize_moe_experts": false, "ray_num_workers": 1, "resources_per_worker": { "GPU": 1 }, "sample_packing_bin_size": 200, "sample_packing_group_size": 100000, "save_only_model": false, "save_safetensors": true, "save_strategy": "epoch", "sequence_len": 32768, "shuffle_before_merging_datasets": false, "shuffle_merged_datasets": true, "skip_prepare_dataset": false, "streaming_multipack_buffer_size": 10000, "strict": false, "tensor_parallel_size": 1, "tf32": false, "tiled_mlp_use_original_mlp": true, "tokenizer_config": "Qwen/Qwen3-8B", "tokenizer_save_jinja_files": true, "torch_dtype": "torch.bfloat16", "train_on_inputs": false, "trl": { "async_prefetch": false, "log_completions": false, "mask_truncated_completions": false, "ref_model_mixup_alpha": 0.9, "ref_model_sync_steps": 64, "replay_buffer_size": 0, "replay_recompute_logps": true, "reroll_max_groups": 1, "reroll_start_fraction": 1.0, "reward_num_workers": 1, "scale_rewards": true, "skip_zero_advantage_batches": true, "sync_ref_model": false, "use_data_producer": false, "use_vllm": false, "vllm_lora_sync": false, "vllm_server_host": "0.0.0.0", "vllm_server_port": 8000 }, "use_otel_metrics": false, "use_ray": false, "val_set_size": 0.0, "vllm": { "device": "auto", "dtype": "auto", "gpu_memory_utilization": 0.9, "host": "0.0.0.0", "port": 8000 }, "wandb_name": "sera-v4-1000-axolotl__Qwen3-8B-v6", "warmup_ratio": 0.1875, "weight_decay": 0.01, "world_size": 4 } [2026-04-24 20:51:03,814] [INFO] [axolotl.cli.checks.check_user_token:37] [PID:293513] Skipping HuggingFace token verification because HF_HUB_OFFLINE is set to True. Only local files will be used. [2026-04-24 20:51:03,972] [DEBUG] [axolotl.utils.config.resolve_dtype:74] [PID:389099] bf16 support detected, enabling for this configuration. [2026-04-24 20:51:04,002] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:389099] baseline 0.000GB () [2026-04-24 20:51:04,002] [INFO] [axolotl.cli.config.load_cfg:341] [PID:389099] config: { "activation_offloading": true, "adam_beta1": 0.9, "adam_beta2": 0.95, "axolotl_config_path": "/e/scratch/jureap59/feuer1/code/axolotl_configs/qwen3_8b_sera_v4_1000_v6.yaml", "base_model": "Qwen/Qwen3-8B", "base_model_config": "Qwen/Qwen3-8B", "batch_size": 32, "bf16": true, "capabilities": { "bf16": true, "compute_capability": "sm_90", "fp8": true, "n_gpu": 4, "n_node": 1, "tf32": true }, "chat_template": "tokenizer_default", "context_parallel_size": 1, "dataloader_num_workers": 4, "dataloader_pin_memory": true, "dataloader_prefetch_factor": 256, "dataset_num_proc": 288, "dataset_prepared_path": "/e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/sera-v4-1000-v6", "datasets": [ { "chat_template": "tokenizer_default", "ds_type": "json", "field_messages": "messages", "message_field_training": "train", "message_property_mappings": { "content": "content", "role": "role" }, "path": "/e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl", "trust_remote_code": false, "type": "chat_template" } ], "ddp": true, "deepspeed": { "bf16": { "enabled": true }, "gradient_accumulation_steps": "auto", "gradient_clipping": "auto", "train_batch_size": "auto", "train_micro_batch_size_per_gpu": "auto", "wall_clock_breakdown": false, "zero_optimization": { "contiguous_gradients": true, "gather_16bit_weights_on_model_save": true, "max_live_parameters": 0, "max_reuse_distance": 0, "overlap_comm": true, "reduce_bucket_size": "auto", "stage": 3, "stage3_param_persistence_threshold": "auto", "stage3_prefetch_bucket_size": "auto", "sub_group_size": 0 } }, "device": "cuda:0", "device_map": { "": 0 }, "dion_rank_fraction": 1.0, "dion_rank_multiple_of": 1, "eaft_alpha": 1.0, "eaft_k": 20, "env_capabilities": { "torch_version": "2.9.1" }, "eval_batch_size": 1, "eval_causal_lm_metrics": [ "sacrebleu", "comet", "ter", "chrf" ], "eval_max_new_tokens": 128, "eval_table_size": 0, "evals_per_epoch": 0, "experimental_skip_move_to_device": true, "flash_attention": true, "fp16": false, "generate_samples": false, "generation_do_sample": true, "generation_max_new_tokens": 50, "generation_prompt_ratio": 0.5, "generation_temperature": 0.7, "gradient_accumulation_steps": 8, "gradient_checkpointing": true, "gradient_checkpointing_kwargs": { "use_reentrant": true }, "include_tkps": true, "layer_offloading": false, "learning_rate": 1e-05, "lisa_layers_attribute": "model.layers", "load_best_model_at_end": false, "load_in_4bit": false, "load_in_8bit": false, "local_rank": 0, "logging_steps": 1, "lora_dropout": 0.0, "loraplus_lr_embedding": 1e-06, "loss_watchdog_patience": 3, "loss_watchdog_threshold": 5.0, "lr_scheduler": "cosine", "max_grad_norm": 1.0, "mean_resizing_embeddings": false, "merge_method": "memory_efficient", "micro_batch_size": 1, "model_config_type": "qwen3", "num_epochs": 6.0, "num_generation_samples": 3, "optimizer": "adamw_torch", "otel_metrics_host": "localhost", "otel_metrics_port": 8000, "output_dir": "/e/data1/datasets/playground/ot-baf/checkpoints/sera-v4-1000-axolotl__Qwen3-8B-v6", "pretrain_multipack_attn": true, "profiler_steps_start": 0, "qlora_sharded_model_loading": false, "quantize_moe_experts": false, "ray_num_workers": 1, "resources_per_worker": { "GPU": 1 }, "sample_packing_bin_size": 200, "sample_packing_group_size": 100000, "save_only_model": false, "save_safetensors": true, "save_strategy": "epoch", "sequence_len": 32768, "shuffle_before_merging_datasets": false, "shuffle_merged_datasets": true, "skip_prepare_dataset": false, "streaming_multipack_buffer_size": 10000, "strict": false, "tensor_parallel_size": 1, "tf32": false, "tiled_mlp_use_original_mlp": true, "tokenizer_config": "Qwen/Qwen3-8B", "tokenizer_save_jinja_files": true, "torch_dtype": "torch.bfloat16", "train_on_inputs": false, "trl": { "async_prefetch": false, "log_completions": false, "mask_truncated_completions": false, "ref_model_mixup_alpha": 0.9, "ref_model_sync_steps": 64, "replay_buffer_size": 0, "replay_recompute_logps": true, "reroll_max_groups": 1, "reroll_start_fraction": 1.0, "reward_num_workers": 1, "scale_rewards": true, "skip_zero_advantage_batches": true, "sync_ref_model": false, "use_data_producer": false, "use_vllm": false, "vllm_lora_sync": false, "vllm_server_host": "0.0.0.0", "vllm_server_port": 8000 }, "use_otel_metrics": false, "use_ray": false, "val_set_size": 0.0, "vllm": { "device": "auto", "dtype": "auto", "gpu_memory_utilization": 0.9, "host": "0.0.0.0", "port": 8000 }, "wandb_name": "sera-v4-1000-axolotl__Qwen3-8B-v6", "warmup_ratio": 0.1875, "weight_decay": 0.01, "world_size": 4 } [2026-04-24 20:51:04,003] [INFO] [axolotl.cli.checks.check_user_token:37] [PID:389099] Skipping HuggingFace token verification because HF_HUB_OFFLINE is set to True. Only local files will be used. [2026-04-24 20:51:04,211] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:293515] Loading raw datasets... [2026-04-24 20:51:04,276] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:293513] EOS: 151645 / <|im_end|> [2026-04-24 20:51:04,277] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:293513] BOS: None / None [2026-04-24 20:51:04,277] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:293513] PAD: 151643 / <|endoftext|> [2026-04-24 20:51:04,277] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:293513] UNK: None / None Generating train split: 0 examples [00:00, ? examples/s][2026-04-24 20:51:04,483] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:389099] EOS: 151645 / <|im_end|> [2026-04-24 20:51:04,485] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:389099] BOS: None / None [2026-04-24 20:51:04,485] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:389099] PAD: 151643 / <|endoftext|> [2026-04-24 20:51:04,485] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:389099] UNK: None / None [2026-04-24 20:51:04,492] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:389102] Loading raw datasets... Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 110 examples [00:00, 1063.19 examples/s] Generating train split: 110 examples [00:00, 700.23 examples/s] Generating train split: 220 examples [00:00, 839.79 examples/s] Generating train split: 220 examples [00:00, 765.21 examples/s] Generating train split: 499 examples [00:00, 1342.34 examples/s] Generating train split: 499 examples [00:00, 1422.37 examples/s] Generating train split: 712 examples [00:00, 1464.18 examples/s] Generating train split: 867 examples [00:00, 1304.67 examples/s] Generating train split: 712 examples [00:00, 1199.03 examples/s] Generating train split: 1000 examples [00:00, 1350.71 examples/s][2026-04-24 20:51:05,234] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:480] [PID:293513] Unable to find prepared dataset in /e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/sera-v4-1000-v6/2e177ea5edfa924e1ca4b12580e4789b [2026-04-24 20:51:05,234] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:293513] Loading raw datasets... [2026-04-24 20:51:05,234] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:293513] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. [2026-04-24 20:51:05,239] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:480] [PID:389099] Unable to find prepared dataset in /e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/sera-v4-1000-v6/2e177ea5edfa924e1ca4b12580e4789b [2026-04-24 20:51:05,241] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:389099] Loading raw datasets... [2026-04-24 20:51:05,241] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:389099] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. [2026-04-24 20:51:05,275] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:293513] Loading dataset: /e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl with base_type: chat_template and prompt_style: None [2026-04-24 20:51:05,276] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:389099] Loading dataset: /e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl with base_type: chat_template and prompt_style: None [2026-04-24 20:51:05,281] [INFO] [axolotl.prompt_strategies.chat_template.__call__:998] [PID:293513] Using chat template: --- {%- if tools %} {{- '<|im_start|>system\n' }} {%- if messages[0].role == 'system' %} {{- messages[0].content + '\n\n' }} {%- endif %} {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} {%- for tool in tools %} {{- "\n" }} {{- tool | tojson }} {%- endfor %} {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} {%- else %} {%- if messages[0].role == 'system' %} {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} {%- endif %} {%- endif %} {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} {%- for message in messages[::-1] %} {%- set index = (messages|length - 1) - loop.index0 %} {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} {%- set ns.multi_step_tool = false %} {%- set ns.last_query_index = index %} {%- endif %} {%- endfor %} {%- for message in messages %} {%- if message.content is string %} {%- set content = message.content %} {%- else %} {%- set content = '' %} {%- endif %} {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} {%- elif message.role == "assistant" %} {%- set reasoning_content = '' %} {%- if message.reasoning_content is string %} {%- set reasoning_content = message.reasoning_content %} {%- else %} {%- if '' in content %} {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} {%- set content = content.split('')[-1].lstrip('\n') %} {%- endif %} {%- endif %} {%- if loop.index0 > ns.last_query_index %} {%- if loop.last or (not loop.last and reasoning_content) %} {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} {%- else %} {{- '<|im_start|>' + message.role + '\n' + content }} {%- endif %} {%- else %} {{- '<|im_start|>' + message.role + '\n' + content }} {%- endif %} {%- if message.tool_calls %} {%- for tool_call in message.tool_calls %} {%- if (loop.first and content) or (not loop.first) %} {{- '\n' }} {%- endif %} {%- if tool_call.function %} {%- set tool_call = tool_call.function %} {%- endif %} {{- '\n{"name": "' }} {{- tool_call.name }} {{- '", "arguments": ' }} {%- if tool_call.arguments is string %} {{- tool_call.arguments }} {%- else %} {{- tool_call.arguments | tojson }} {%- endif %} {{- '}\n' }} {%- endfor %} {%- endif %} {{- '<|im_end|>\n' }} {%- elif message.role == "tool" %} {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} {{- '<|im_start|>user' }} {%- endif %} {{- '\n\n' }} {{- content }} {{- '\n' }} {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} {{- '<|im_end|>\n' }} {%- endif %} {%- endif %} {%- endfor %} {%- if add_generation_prompt %} {{- '<|im_start|>assistant\n' }} {%- if enable_thinking is defined and enable_thinking is false %} {{- '\n\n\n\n' }} {%- endif %} {%- endif %} --- Generating train split: 1000 examples [00:00, 1611.64 examples/s][2026-04-24 20:51:05,282] [INFO] [axolotl.prompt_strategies.chat_template.__call__:998] [PID:389099] Using chat template: --- {%- if tools %} {{- '<|im_start|>system\n' }} {%- if messages[0].role == 'system' %} {{- messages[0].content + '\n\n' }} {%- endif %} {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} {%- for tool in tools %} {{- "\n" }} {{- tool | tojson }} {%- endfor %} {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} {%- else %} {%- if messages[0].role == 'system' %} {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} {%- endif %} {%- endif %} {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} {%- for message in messages[::-1] %} {%- set index = (messages|length - 1) - loop.index0 %} {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} {%- set ns.multi_step_tool = false %} {%- set ns.last_query_index = index %} {%- endif %} {%- endfor %} {%- for message in messages %} {%- if message.content is string %} {%- set content = message.content %} {%- else %} {%- set content = '' %} {%- endif %} {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} {%- elif message.role == "assistant" %} {%- set reasoning_content = '' %} {%- if message.reasoning_content is string %} {%- set reasoning_content = message.reasoning_content %} {%- else %} {%- if '' in content %} {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} {%- set content = content.split('')[-1].lstrip('\n') %} {%- endif %} {%- endif %} {%- if loop.index0 > ns.last_query_index %} {%- if loop.last or (not loop.last and reasoning_content) %} {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} {%- else %} {{- '<|im_start|>' + message.role + '\n' + content }} {%- endif %} {%- else %} {{- '<|im_start|>' + message.role + '\n' + content }} {%- endif %} {%- if message.tool_calls %} {%- for tool_call in message.tool_calls %} {%- if (loop.first and content) or (not loop.first) %} {{- '\n' }} {%- endif %} {%- if tool_call.function %} {%- set tool_call = tool_call.function %} {%- endif %} {{- '\n{"name": "' }} {{- tool_call.name }} {{- '", "arguments": ' }} {%- if tool_call.arguments is string %} {{- tool_call.arguments }} {%- else %} {{- tool_call.arguments | tojson }} {%- endif %} {{- '}\n' }} {%- endfor %} {%- endif %} {{- '<|im_end|>\n' }} {%- elif message.role == "tool" %} {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} {{- '<|im_start|>user' }} {%- endif %} {{- '\n\n' }} {{- content }} {{- '\n' }} {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} {{- '<|im_end|>\n' }} {%- endif %} {%- endif %} {%- endfor %} {%- if add_generation_prompt %} {{- '<|im_start|>assistant\n' }} {%- if enable_thinking is defined and enable_thinking is false %} {{- '\n\n\n\n' }} {%- endif %} {%- endif %} --- [2026-04-24 20:51:05,285] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:293514] Loading raw datasets... [2026-04-24 20:51:05,291] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:389100] Loading raw datasets... Generating train split: 1000 examples [00:00, 1356.77 examples/s] [rank3]: Traceback (most recent call last): [rank3]: File "", line 198, in _run_module_as_main [rank3]: File "", line 88, in _run_code [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 124, in [rank3]: fire.Fire(do_cli) [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 135, in Fire [rank3]: component_trace = _Fire(component, args, parsed_flag_args, context, name) [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 468, in _Fire [rank3]: component, remaining_args = _CallAndUpdateTrace( [rank3]: ^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace [rank3]: component = fn(*varargs, **kwargs) [rank3]: ^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 91, in do_cli [rank3]: do_train(parsed_cfg, parsed_cli_args) [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 43, in do_train [rank3]: dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args) [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/telemetry/errors.py", line 124, in wrapper [rank3]: return func(*args, **kwargs) [rank3]: ^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/common/datasets.py", line 61, in load_datasets [rank3]: train_dataset, eval_dataset, total_num_steps, prompters = prepare_datasets( [rank3]: ^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/utils.py", line 50, in wrapper [rank3]: return func(*args, **kwargs) [rank3]: ^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 65, in prepare_datasets [rank3]: return _prepare_standard_dataset(cfg, tokenizer, processor) [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 98, in _prepare_standard_dataset [rank3]: train_dataset, eval_dataset, prompters = loader.load(_load_datasets) [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/lock.py", line 38, in load [rank3]: result = load_fn() [rank3]: ^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 77, in _load_datasets [rank3]: train_dataset, eval_dataset, prompters = _load_and_prepare_datasets( [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 496, in _load_and_prepare_datasets [rank3]: dataset, prompters = _load_tokenized_prepared_datasets( [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 299, in _load_tokenized_prepared_datasets [rank3]: dataset, prompters = _load_raw_datasets( [rank3]: ^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 331, in _load_raw_datasets [rank3]: dataset_wrapper, dataset_prompter = _load_and_process_single_dataset( [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 384, in _load_and_process_single_dataset [rank3]: dataset = load_dataset_with_config( [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/shared.py", line 118, in load_dataset_with_config [rank3]: return _load_from_local_path(dataset_config, load_dataset_kwargs) [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/shared.py", line 247, in _load_from_local_path [rank3]: return load_dataset( [rank3]: ^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/datasets/load.py", line 1508, in load_dataset [rank3]: builder_instance.download_and_prepare( [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/datasets/builder.py", line 884, in download_and_prepare [rank3]: self._download_and_prepare( [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/datasets/builder.py", line 949, in _download_and_prepare [rank3]: raise OSError("Cannot find data file. " + "\nOriginal error:\n" + str(e)) from None [rank3]: OSError: Cannot find data file. [rank3]: Original error: [rank3]: [Errno 2] No such file or directory: '/e/data1/datasets/playground/ot-baf/hf_hub/datasets/json/default-b49d7fb735d1a400/0.0.0/ff8895240c1c6d49c1ae5a8af7cd4b065d168a612f3f6bd42470270b78a42c18.incomplete/json-train-00000-00000-of-NNNNN.arrow' [rank2]: Traceback (most recent call last): [rank2]: File "", line 198, in _run_module_as_main [rank2]: File "", line 88, in _run_code [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 124, in [rank2]: fire.Fire(do_cli) [rank2]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 135, in Fire [rank2]: component_trace = _Fire(component, args, parsed_flag_args, context, name) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 468, in _Fire [rank2]: component, remaining_args = _CallAndUpdateTrace( [rank2]: ^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace [rank2]: component = fn(*varargs, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 91, in do_cli [rank2]: do_train(parsed_cfg, parsed_cli_args) [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 43, in do_train [rank2]: dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/telemetry/errors.py", line 124, in wrapper [rank2]: return func(*args, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/common/datasets.py", line 61, in load_datasets [rank2]: train_dataset, eval_dataset, total_num_steps, prompters = prepare_datasets( [rank2]: ^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/utils.py", line 50, in wrapper [rank2]: return func(*args, **kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 65, in prepare_datasets [rank2]: return _prepare_standard_dataset(cfg, tokenizer, processor) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 98, in _prepare_standard_dataset [rank2]: train_dataset, eval_dataset, prompters = loader.load(_load_datasets) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/lock.py", line 38, in load [rank2]: result = load_fn() [rank2]: ^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 77, in _load_datasets [rank2]: train_dataset, eval_dataset, prompters = _load_and_prepare_datasets( [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 496, in _load_and_prepare_datasets [rank2]: dataset, prompters = _load_tokenized_prepared_datasets( [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 299, in _load_tokenized_prepared_datasets [rank2]: dataset, prompters = _load_raw_datasets( [rank2]: ^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 331, in _load_raw_datasets [rank2]: dataset_wrapper, dataset_prompter = _load_and_process_single_dataset( [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 384, in _load_and_process_single_dataset [rank2]: dataset = load_dataset_with_config( [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/shared.py", line 118, in load_dataset_with_config [rank2]: return _load_from_local_path(dataset_config, load_dataset_kwargs) [rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/shared.py", line 247, in _load_from_local_path [rank2]: return load_dataset( [rank2]: ^^^^^^^^^^^^^ [rank2]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/datasets/load.py", line 1508, in load_dataset [rank2]: builder_instance.download_and_prepare( [rank2]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/datasets/builder.py", line 884, in download_and_prepare [rank2]: self._download_and_prepare( [rank2]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/datasets/builder.py", line 949, in _download_and_prepare [rank2]: raise OSError("Cannot find data file. " + "\nOriginal error:\n" + str(e)) from None [rank2]: OSError: Cannot find data file. [rank2]: Original error: [rank2]: [Errno 2] No such file or directory: '/e/data1/datasets/playground/ot-baf/hf_hub/datasets/json/default-b49d7fb735d1a400/0.0.0/ff8895240c1c6d49c1ae5a8af7cd4b065d168a612f3f6bd42470270b78a42c18.incomplete/json-train-00000-00000-of-NNNNN.arrow' [2026-04-24 20:51:05,311] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:293514] Loading dataset: /e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl with base_type: chat_template and prompt_style: None [2026-04-24 20:51:05,316] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:389100] Loading dataset: /e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl with base_type: chat_template and prompt_style: None [2026-04-24 20:51:05,343] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:389101] Loading raw datasets... [2026-04-24 20:51:05,375] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:389101] Loading dataset: /e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl with base_type: chat_template and prompt_style: None