717 lines
36 KiB
Plaintext
717 lines
36 KiB
Plaintext
[2026-04-24 20:51:03,702] [DEBUG] [axolotl.utils.config.resolve_dtype:74] [PID:293513] bf16 support detected, enabling for this configuration.
|
|
[2026-04-24 20:51:03,717] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:293513] baseline 0.000GB ()
|
|
[2026-04-24 20:51:03,718] [INFO] [axolotl.cli.config.load_cfg:341] [PID:293513] config:
|
|
{
|
|
"activation_offloading": true,
|
|
"adam_beta1": 0.9,
|
|
"adam_beta2": 0.95,
|
|
"axolotl_config_path": "/e/scratch/jureap59/feuer1/code/axolotl_configs/qwen3_8b_sera_v4_1000_v6.yaml",
|
|
"base_model": "Qwen/Qwen3-8B",
|
|
"base_model_config": "Qwen/Qwen3-8B",
|
|
"batch_size": 32,
|
|
"bf16": true,
|
|
"capabilities": {
|
|
"bf16": true,
|
|
"compute_capability": "sm_90",
|
|
"fp8": true,
|
|
"n_gpu": 4,
|
|
"n_node": 1,
|
|
"tf32": true
|
|
},
|
|
"chat_template": "tokenizer_default",
|
|
"context_parallel_size": 1,
|
|
"dataloader_num_workers": 4,
|
|
"dataloader_pin_memory": true,
|
|
"dataloader_prefetch_factor": 256,
|
|
"dataset_num_proc": 288,
|
|
"dataset_prepared_path": "/e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/sera-v4-1000-v6",
|
|
"datasets": [
|
|
{
|
|
"chat_template": "tokenizer_default",
|
|
"ds_type": "json",
|
|
"field_messages": "messages",
|
|
"message_field_training": "train",
|
|
"message_property_mappings": {
|
|
"content": "content",
|
|
"role": "role"
|
|
},
|
|
"path": "/e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl",
|
|
"trust_remote_code": false,
|
|
"type": "chat_template"
|
|
}
|
|
],
|
|
"ddp": true,
|
|
"deepspeed": {
|
|
"bf16": {
|
|
"enabled": true
|
|
},
|
|
"gradient_accumulation_steps": "auto",
|
|
"gradient_clipping": "auto",
|
|
"train_batch_size": "auto",
|
|
"train_micro_batch_size_per_gpu": "auto",
|
|
"wall_clock_breakdown": false,
|
|
"zero_optimization": {
|
|
"contiguous_gradients": true,
|
|
"gather_16bit_weights_on_model_save": true,
|
|
"max_live_parameters": 0,
|
|
"max_reuse_distance": 0,
|
|
"overlap_comm": true,
|
|
"reduce_bucket_size": "auto",
|
|
"stage": 3,
|
|
"stage3_param_persistence_threshold": "auto",
|
|
"stage3_prefetch_bucket_size": "auto",
|
|
"sub_group_size": 0
|
|
}
|
|
},
|
|
"device": "cuda:0",
|
|
"device_map": {
|
|
"": 0
|
|
},
|
|
"dion_rank_fraction": 1.0,
|
|
"dion_rank_multiple_of": 1,
|
|
"eaft_alpha": 1.0,
|
|
"eaft_k": 20,
|
|
"env_capabilities": {
|
|
"torch_version": "2.9.1"
|
|
},
|
|
"eval_batch_size": 1,
|
|
"eval_causal_lm_metrics": [
|
|
"sacrebleu",
|
|
"comet",
|
|
"ter",
|
|
"chrf"
|
|
],
|
|
"eval_max_new_tokens": 128,
|
|
"eval_table_size": 0,
|
|
"evals_per_epoch": 0,
|
|
"experimental_skip_move_to_device": true,
|
|
"flash_attention": true,
|
|
"fp16": false,
|
|
"generate_samples": false,
|
|
"generation_do_sample": true,
|
|
"generation_max_new_tokens": 50,
|
|
"generation_prompt_ratio": 0.5,
|
|
"generation_temperature": 0.7,
|
|
"gradient_accumulation_steps": 8,
|
|
"gradient_checkpointing": true,
|
|
"gradient_checkpointing_kwargs": {
|
|
"use_reentrant": true
|
|
},
|
|
"include_tkps": true,
|
|
"layer_offloading": false,
|
|
"learning_rate": 1e-05,
|
|
"lisa_layers_attribute": "model.layers",
|
|
"load_best_model_at_end": false,
|
|
"load_in_4bit": false,
|
|
"load_in_8bit": false,
|
|
"local_rank": 0,
|
|
"logging_steps": 1,
|
|
"lora_dropout": 0.0,
|
|
"loraplus_lr_embedding": 1e-06,
|
|
"loss_watchdog_patience": 3,
|
|
"loss_watchdog_threshold": 5.0,
|
|
"lr_scheduler": "cosine",
|
|
"max_grad_norm": 1.0,
|
|
"mean_resizing_embeddings": false,
|
|
"merge_method": "memory_efficient",
|
|
"micro_batch_size": 1,
|
|
"model_config_type": "qwen3",
|
|
"num_epochs": 6.0,
|
|
"num_generation_samples": 3,
|
|
"optimizer": "adamw_torch",
|
|
"otel_metrics_host": "localhost",
|
|
"otel_metrics_port": 8000,
|
|
"output_dir": "/e/data1/datasets/playground/ot-baf/checkpoints/sera-v4-1000-axolotl__Qwen3-8B-v6",
|
|
"pretrain_multipack_attn": true,
|
|
"profiler_steps_start": 0,
|
|
"qlora_sharded_model_loading": false,
|
|
"quantize_moe_experts": false,
|
|
"ray_num_workers": 1,
|
|
"resources_per_worker": {
|
|
"GPU": 1
|
|
},
|
|
"sample_packing_bin_size": 200,
|
|
"sample_packing_group_size": 100000,
|
|
"save_only_model": false,
|
|
"save_safetensors": true,
|
|
"save_strategy": "epoch",
|
|
"sequence_len": 32768,
|
|
"shuffle_before_merging_datasets": false,
|
|
"shuffle_merged_datasets": true,
|
|
"skip_prepare_dataset": false,
|
|
"streaming_multipack_buffer_size": 10000,
|
|
"strict": false,
|
|
"tensor_parallel_size": 1,
|
|
"tf32": false,
|
|
"tiled_mlp_use_original_mlp": true,
|
|
"tokenizer_config": "Qwen/Qwen3-8B",
|
|
"tokenizer_save_jinja_files": true,
|
|
"torch_dtype": "torch.bfloat16",
|
|
"train_on_inputs": false,
|
|
"trl": {
|
|
"async_prefetch": false,
|
|
"log_completions": false,
|
|
"mask_truncated_completions": false,
|
|
"ref_model_mixup_alpha": 0.9,
|
|
"ref_model_sync_steps": 64,
|
|
"replay_buffer_size": 0,
|
|
"replay_recompute_logps": true,
|
|
"reroll_max_groups": 1,
|
|
"reroll_start_fraction": 1.0,
|
|
"reward_num_workers": 1,
|
|
"scale_rewards": true,
|
|
"skip_zero_advantage_batches": true,
|
|
"sync_ref_model": false,
|
|
"use_data_producer": false,
|
|
"use_vllm": false,
|
|
"vllm_lora_sync": false,
|
|
"vllm_server_host": "0.0.0.0",
|
|
"vllm_server_port": 8000
|
|
},
|
|
"use_otel_metrics": false,
|
|
"use_ray": false,
|
|
"val_set_size": 0.0,
|
|
"vllm": {
|
|
"device": "auto",
|
|
"dtype": "auto",
|
|
"gpu_memory_utilization": 0.9,
|
|
"host": "0.0.0.0",
|
|
"port": 8000
|
|
},
|
|
"wandb_name": "sera-v4-1000-axolotl__Qwen3-8B-v6",
|
|
"warmup_ratio": 0.1875,
|
|
"weight_decay": 0.01,
|
|
"world_size": 4
|
|
}
|
|
[2026-04-24 20:51:03,814] [INFO] [axolotl.cli.checks.check_user_token:37] [PID:293513] Skipping HuggingFace token verification because HF_HUB_OFFLINE is set to True. Only local files will be used.
|
|
[2026-04-24 20:51:03,972] [DEBUG] [axolotl.utils.config.resolve_dtype:74] [PID:389099] bf16 support detected, enabling for this configuration.
|
|
[2026-04-24 20:51:04,002] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:389099] baseline 0.000GB ()
|
|
[2026-04-24 20:51:04,002] [INFO] [axolotl.cli.config.load_cfg:341] [PID:389099] config:
|
|
{
|
|
"activation_offloading": true,
|
|
"adam_beta1": 0.9,
|
|
"adam_beta2": 0.95,
|
|
"axolotl_config_path": "/e/scratch/jureap59/feuer1/code/axolotl_configs/qwen3_8b_sera_v4_1000_v6.yaml",
|
|
"base_model": "Qwen/Qwen3-8B",
|
|
"base_model_config": "Qwen/Qwen3-8B",
|
|
"batch_size": 32,
|
|
"bf16": true,
|
|
"capabilities": {
|
|
"bf16": true,
|
|
"compute_capability": "sm_90",
|
|
"fp8": true,
|
|
"n_gpu": 4,
|
|
"n_node": 1,
|
|
"tf32": true
|
|
},
|
|
"chat_template": "tokenizer_default",
|
|
"context_parallel_size": 1,
|
|
"dataloader_num_workers": 4,
|
|
"dataloader_pin_memory": true,
|
|
"dataloader_prefetch_factor": 256,
|
|
"dataset_num_proc": 288,
|
|
"dataset_prepared_path": "/e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/sera-v4-1000-v6",
|
|
"datasets": [
|
|
{
|
|
"chat_template": "tokenizer_default",
|
|
"ds_type": "json",
|
|
"field_messages": "messages",
|
|
"message_field_training": "train",
|
|
"message_property_mappings": {
|
|
"content": "content",
|
|
"role": "role"
|
|
},
|
|
"path": "/e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl",
|
|
"trust_remote_code": false,
|
|
"type": "chat_template"
|
|
}
|
|
],
|
|
"ddp": true,
|
|
"deepspeed": {
|
|
"bf16": {
|
|
"enabled": true
|
|
},
|
|
"gradient_accumulation_steps": "auto",
|
|
"gradient_clipping": "auto",
|
|
"train_batch_size": "auto",
|
|
"train_micro_batch_size_per_gpu": "auto",
|
|
"wall_clock_breakdown": false,
|
|
"zero_optimization": {
|
|
"contiguous_gradients": true,
|
|
"gather_16bit_weights_on_model_save": true,
|
|
"max_live_parameters": 0,
|
|
"max_reuse_distance": 0,
|
|
"overlap_comm": true,
|
|
"reduce_bucket_size": "auto",
|
|
"stage": 3,
|
|
"stage3_param_persistence_threshold": "auto",
|
|
"stage3_prefetch_bucket_size": "auto",
|
|
"sub_group_size": 0
|
|
}
|
|
},
|
|
"device": "cuda:0",
|
|
"device_map": {
|
|
"": 0
|
|
},
|
|
"dion_rank_fraction": 1.0,
|
|
"dion_rank_multiple_of": 1,
|
|
"eaft_alpha": 1.0,
|
|
"eaft_k": 20,
|
|
"env_capabilities": {
|
|
"torch_version": "2.9.1"
|
|
},
|
|
"eval_batch_size": 1,
|
|
"eval_causal_lm_metrics": [
|
|
"sacrebleu",
|
|
"comet",
|
|
"ter",
|
|
"chrf"
|
|
],
|
|
"eval_max_new_tokens": 128,
|
|
"eval_table_size": 0,
|
|
"evals_per_epoch": 0,
|
|
"experimental_skip_move_to_device": true,
|
|
"flash_attention": true,
|
|
"fp16": false,
|
|
"generate_samples": false,
|
|
"generation_do_sample": true,
|
|
"generation_max_new_tokens": 50,
|
|
"generation_prompt_ratio": 0.5,
|
|
"generation_temperature": 0.7,
|
|
"gradient_accumulation_steps": 8,
|
|
"gradient_checkpointing": true,
|
|
"gradient_checkpointing_kwargs": {
|
|
"use_reentrant": true
|
|
},
|
|
"include_tkps": true,
|
|
"layer_offloading": false,
|
|
"learning_rate": 1e-05,
|
|
"lisa_layers_attribute": "model.layers",
|
|
"load_best_model_at_end": false,
|
|
"load_in_4bit": false,
|
|
"load_in_8bit": false,
|
|
"local_rank": 0,
|
|
"logging_steps": 1,
|
|
"lora_dropout": 0.0,
|
|
"loraplus_lr_embedding": 1e-06,
|
|
"loss_watchdog_patience": 3,
|
|
"loss_watchdog_threshold": 5.0,
|
|
"lr_scheduler": "cosine",
|
|
"max_grad_norm": 1.0,
|
|
"mean_resizing_embeddings": false,
|
|
"merge_method": "memory_efficient",
|
|
"micro_batch_size": 1,
|
|
"model_config_type": "qwen3",
|
|
"num_epochs": 6.0,
|
|
"num_generation_samples": 3,
|
|
"optimizer": "adamw_torch",
|
|
"otel_metrics_host": "localhost",
|
|
"otel_metrics_port": 8000,
|
|
"output_dir": "/e/data1/datasets/playground/ot-baf/checkpoints/sera-v4-1000-axolotl__Qwen3-8B-v6",
|
|
"pretrain_multipack_attn": true,
|
|
"profiler_steps_start": 0,
|
|
"qlora_sharded_model_loading": false,
|
|
"quantize_moe_experts": false,
|
|
"ray_num_workers": 1,
|
|
"resources_per_worker": {
|
|
"GPU": 1
|
|
},
|
|
"sample_packing_bin_size": 200,
|
|
"sample_packing_group_size": 100000,
|
|
"save_only_model": false,
|
|
"save_safetensors": true,
|
|
"save_strategy": "epoch",
|
|
"sequence_len": 32768,
|
|
"shuffle_before_merging_datasets": false,
|
|
"shuffle_merged_datasets": true,
|
|
"skip_prepare_dataset": false,
|
|
"streaming_multipack_buffer_size": 10000,
|
|
"strict": false,
|
|
"tensor_parallel_size": 1,
|
|
"tf32": false,
|
|
"tiled_mlp_use_original_mlp": true,
|
|
"tokenizer_config": "Qwen/Qwen3-8B",
|
|
"tokenizer_save_jinja_files": true,
|
|
"torch_dtype": "torch.bfloat16",
|
|
"train_on_inputs": false,
|
|
"trl": {
|
|
"async_prefetch": false,
|
|
"log_completions": false,
|
|
"mask_truncated_completions": false,
|
|
"ref_model_mixup_alpha": 0.9,
|
|
"ref_model_sync_steps": 64,
|
|
"replay_buffer_size": 0,
|
|
"replay_recompute_logps": true,
|
|
"reroll_max_groups": 1,
|
|
"reroll_start_fraction": 1.0,
|
|
"reward_num_workers": 1,
|
|
"scale_rewards": true,
|
|
"skip_zero_advantage_batches": true,
|
|
"sync_ref_model": false,
|
|
"use_data_producer": false,
|
|
"use_vllm": false,
|
|
"vllm_lora_sync": false,
|
|
"vllm_server_host": "0.0.0.0",
|
|
"vllm_server_port": 8000
|
|
},
|
|
"use_otel_metrics": false,
|
|
"use_ray": false,
|
|
"val_set_size": 0.0,
|
|
"vllm": {
|
|
"device": "auto",
|
|
"dtype": "auto",
|
|
"gpu_memory_utilization": 0.9,
|
|
"host": "0.0.0.0",
|
|
"port": 8000
|
|
},
|
|
"wandb_name": "sera-v4-1000-axolotl__Qwen3-8B-v6",
|
|
"warmup_ratio": 0.1875,
|
|
"weight_decay": 0.01,
|
|
"world_size": 4
|
|
}
|
|
[2026-04-24 20:51:04,003] [INFO] [axolotl.cli.checks.check_user_token:37] [PID:389099] Skipping HuggingFace token verification because HF_HUB_OFFLINE is set to True. Only local files will be used.
|
|
[2026-04-24 20:51:04,211] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:293515] Loading raw datasets...
|
|
[2026-04-24 20:51:04,276] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:293513] EOS: 151645 / <|im_end|>
|
|
[2026-04-24 20:51:04,277] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:293513] BOS: None / None
|
|
[2026-04-24 20:51:04,277] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:293513] PAD: 151643 / <|endoftext|>
|
|
[2026-04-24 20:51:04,277] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:293513] UNK: None / None
|
|
|
|
Generating train split: 0 examples [00:00, ? examples/s][2026-04-24 20:51:04,483] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:389099] EOS: 151645 / <|im_end|>
|
|
[2026-04-24 20:51:04,485] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:389099] BOS: None / None
|
|
[2026-04-24 20:51:04,485] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:389099] PAD: 151643 / <|endoftext|>
|
|
[2026-04-24 20:51:04,485] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:389099] UNK: None / None
|
|
[2026-04-24 20:51:04,492] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:389102] Loading raw datasets...
|
|
|
|
Generating train split: 0 examples [00:00, ? examples/s]
|
|
Generating train split: 110 examples [00:00, 1063.19 examples/s]
|
|
Generating train split: 110 examples [00:00, 700.23 examples/s]
|
|
Generating train split: 220 examples [00:00, 839.79 examples/s]
|
|
Generating train split: 220 examples [00:00, 765.21 examples/s]
|
|
Generating train split: 499 examples [00:00, 1342.34 examples/s]
|
|
Generating train split: 499 examples [00:00, 1422.37 examples/s]
|
|
Generating train split: 712 examples [00:00, 1464.18 examples/s]
|
|
Generating train split: 867 examples [00:00, 1304.67 examples/s]
|
|
Generating train split: 712 examples [00:00, 1199.03 examples/s]
|
|
Generating train split: 1000 examples [00:00, 1350.71 examples/s][2026-04-24 20:51:05,234] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:480] [PID:293513] Unable to find prepared dataset in /e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/sera-v4-1000-v6/2e177ea5edfa924e1ca4b12580e4789b
|
|
[2026-04-24 20:51:05,234] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:293513] Loading raw datasets...
|
|
[2026-04-24 20:51:05,234] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:293513] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
|
|
[2026-04-24 20:51:05,239] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:480] [PID:389099] Unable to find prepared dataset in /e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/sera-v4-1000-v6/2e177ea5edfa924e1ca4b12580e4789b
|
|
[2026-04-24 20:51:05,241] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:389099] Loading raw datasets...
|
|
[2026-04-24 20:51:05,241] [WARNING] [axolotl.utils.data.sft._load_raw_datasets:322] [PID:389099] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`.
|
|
[2026-04-24 20:51:05,275] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:293513] Loading dataset: /e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl with base_type: chat_template and prompt_style: None
|
|
[2026-04-24 20:51:05,276] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:389099] Loading dataset: /e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl with base_type: chat_template and prompt_style: None
|
|
[2026-04-24 20:51:05,281] [INFO] [axolotl.prompt_strategies.chat_template.__call__:998] [PID:293513] Using chat template:
|
|
---
|
|
{%- if tools %}
|
|
{{- '<|im_start|>system\n' }}
|
|
{%- if messages[0].role == 'system' %}
|
|
{{- messages[0].content + '\n\n' }}
|
|
{%- endif %}
|
|
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
|
{%- for tool in tools %}
|
|
{{- "\n" }}
|
|
{{- tool | tojson }}
|
|
{%- endfor %}
|
|
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
|
{%- else %}
|
|
{%- if messages[0].role == 'system' %}
|
|
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
|
{%- for message in messages[::-1] %}
|
|
{%- set index = (messages|length - 1) - loop.index0 %}
|
|
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
|
|
{%- set ns.multi_step_tool = false %}
|
|
{%- set ns.last_query_index = index %}
|
|
{%- endif %}
|
|
{%- endfor %}
|
|
{%- for message in messages %}
|
|
{%- if message.content is string %}
|
|
{%- set content = message.content %}
|
|
{%- else %}
|
|
{%- set content = '' %}
|
|
{%- endif %}
|
|
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
|
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
|
{%- elif message.role == "assistant" %}
|
|
{%- set reasoning_content = '' %}
|
|
{%- if message.reasoning_content is string %}
|
|
{%- set reasoning_content = message.reasoning_content %}
|
|
{%- else %}
|
|
{%- if '</think>' in content %}
|
|
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
|
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
{%- if loop.index0 > ns.last_query_index %}
|
|
{%- if loop.last or (not loop.last and reasoning_content) %}
|
|
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
|
|
{%- else %}
|
|
{{- '<|im_start|>' + message.role + '\n' + content }}
|
|
{%- endif %}
|
|
{%- else %}
|
|
{{- '<|im_start|>' + message.role + '\n' + content }}
|
|
{%- endif %}
|
|
{%- if message.tool_calls %}
|
|
{%- for tool_call in message.tool_calls %}
|
|
{%- if (loop.first and content) or (not loop.first) %}
|
|
{{- '\n' }}
|
|
{%- endif %}
|
|
{%- if tool_call.function %}
|
|
{%- set tool_call = tool_call.function %}
|
|
{%- endif %}
|
|
{{- '<tool_call>\n{"name": "' }}
|
|
{{- tool_call.name }}
|
|
{{- '", "arguments": ' }}
|
|
{%- if tool_call.arguments is string %}
|
|
{{- tool_call.arguments }}
|
|
{%- else %}
|
|
{{- tool_call.arguments | tojson }}
|
|
{%- endif %}
|
|
{{- '}\n</tool_call>' }}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
{{- '<|im_end|>\n' }}
|
|
{%- elif message.role == "tool" %}
|
|
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
|
{{- '<|im_start|>user' }}
|
|
{%- endif %}
|
|
{{- '\n<tool_response>\n' }}
|
|
{{- content }}
|
|
{{- '\n</tool_response>' }}
|
|
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
|
{{- '<|im_end|>\n' }}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
{%- endfor %}
|
|
{%- if add_generation_prompt %}
|
|
{{- '<|im_start|>assistant\n' }}
|
|
{%- if enable_thinking is defined and enable_thinking is false %}
|
|
{{- '<think>\n\n</think>\n\n' }}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
---
|
|
|
|
Generating train split: 1000 examples [00:00, 1611.64 examples/s][2026-04-24 20:51:05,282] [INFO] [axolotl.prompt_strategies.chat_template.__call__:998] [PID:389099] Using chat template:
|
|
---
|
|
{%- if tools %}
|
|
{{- '<|im_start|>system\n' }}
|
|
{%- if messages[0].role == 'system' %}
|
|
{{- messages[0].content + '\n\n' }}
|
|
{%- endif %}
|
|
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
|
{%- for tool in tools %}
|
|
{{- "\n" }}
|
|
{{- tool | tojson }}
|
|
{%- endfor %}
|
|
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
|
{%- else %}
|
|
{%- if messages[0].role == 'system' %}
|
|
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
|
{%- for message in messages[::-1] %}
|
|
{%- set index = (messages|length - 1) - loop.index0 %}
|
|
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
|
|
{%- set ns.multi_step_tool = false %}
|
|
{%- set ns.last_query_index = index %}
|
|
{%- endif %}
|
|
{%- endfor %}
|
|
{%- for message in messages %}
|
|
{%- if message.content is string %}
|
|
{%- set content = message.content %}
|
|
{%- else %}
|
|
{%- set content = '' %}
|
|
{%- endif %}
|
|
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
|
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
|
{%- elif message.role == "assistant" %}
|
|
{%- set reasoning_content = '' %}
|
|
{%- if message.reasoning_content is string %}
|
|
{%- set reasoning_content = message.reasoning_content %}
|
|
{%- else %}
|
|
{%- if '</think>' in content %}
|
|
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
|
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
{%- if loop.index0 > ns.last_query_index %}
|
|
{%- if loop.last or (not loop.last and reasoning_content) %}
|
|
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
|
|
{%- else %}
|
|
{{- '<|im_start|>' + message.role + '\n' + content }}
|
|
{%- endif %}
|
|
{%- else %}
|
|
{{- '<|im_start|>' + message.role + '\n' + content }}
|
|
{%- endif %}
|
|
{%- if message.tool_calls %}
|
|
{%- for tool_call in message.tool_calls %}
|
|
{%- if (loop.first and content) or (not loop.first) %}
|
|
{{- '\n' }}
|
|
{%- endif %}
|
|
{%- if tool_call.function %}
|
|
{%- set tool_call = tool_call.function %}
|
|
{%- endif %}
|
|
{{- '<tool_call>\n{"name": "' }}
|
|
{{- tool_call.name }}
|
|
{{- '", "arguments": ' }}
|
|
{%- if tool_call.arguments is string %}
|
|
{{- tool_call.arguments }}
|
|
{%- else %}
|
|
{{- tool_call.arguments | tojson }}
|
|
{%- endif %}
|
|
{{- '}\n</tool_call>' }}
|
|
{%- endfor %}
|
|
{%- endif %}
|
|
{{- '<|im_end|>\n' }}
|
|
{%- elif message.role == "tool" %}
|
|
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
|
{{- '<|im_start|>user' }}
|
|
{%- endif %}
|
|
{{- '\n<tool_response>\n' }}
|
|
{{- content }}
|
|
{{- '\n</tool_response>' }}
|
|
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
|
{{- '<|im_end|>\n' }}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
{%- endfor %}
|
|
{%- if add_generation_prompt %}
|
|
{{- '<|im_start|>assistant\n' }}
|
|
{%- if enable_thinking is defined and enable_thinking is false %}
|
|
{{- '<think>\n\n</think>\n\n' }}
|
|
{%- endif %}
|
|
{%- endif %}
|
|
---
|
|
[2026-04-24 20:51:05,285] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:293514] Loading raw datasets...
|
|
[2026-04-24 20:51:05,291] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:389100] Loading raw datasets...
|
|
|
|
Generating train split: 1000 examples [00:00, 1356.77 examples/s]
|
|
[rank3]: Traceback (most recent call last):
|
|
[rank3]: File "<frozen runpy>", line 198, in _run_module_as_main
|
|
[rank3]: File "<frozen runpy>", line 88, in _run_code
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 124, in <module>
|
|
[rank3]: fire.Fire(do_cli)
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 135, in Fire
|
|
[rank3]: component_trace = _Fire(component, args, parsed_flag_args, context, name)
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 468, in _Fire
|
|
[rank3]: component, remaining_args = _CallAndUpdateTrace(
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace
|
|
[rank3]: component = fn(*varargs, **kwargs)
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 91, in do_cli
|
|
[rank3]: do_train(parsed_cfg, parsed_cli_args)
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 43, in do_train
|
|
[rank3]: dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/telemetry/errors.py", line 124, in wrapper
|
|
[rank3]: return func(*args, **kwargs)
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/common/datasets.py", line 61, in load_datasets
|
|
[rank3]: train_dataset, eval_dataset, total_num_steps, prompters = prepare_datasets(
|
|
[rank3]: ^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/utils.py", line 50, in wrapper
|
|
[rank3]: return func(*args, **kwargs)
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 65, in prepare_datasets
|
|
[rank3]: return _prepare_standard_dataset(cfg, tokenizer, processor)
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 98, in _prepare_standard_dataset
|
|
[rank3]: train_dataset, eval_dataset, prompters = loader.load(_load_datasets)
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/lock.py", line 38, in load
|
|
[rank3]: result = load_fn()
|
|
[rank3]: ^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 77, in _load_datasets
|
|
[rank3]: train_dataset, eval_dataset, prompters = _load_and_prepare_datasets(
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 496, in _load_and_prepare_datasets
|
|
[rank3]: dataset, prompters = _load_tokenized_prepared_datasets(
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 299, in _load_tokenized_prepared_datasets
|
|
[rank3]: dataset, prompters = _load_raw_datasets(
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 331, in _load_raw_datasets
|
|
[rank3]: dataset_wrapper, dataset_prompter = _load_and_process_single_dataset(
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 384, in _load_and_process_single_dataset
|
|
[rank3]: dataset = load_dataset_with_config(
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/shared.py", line 118, in load_dataset_with_config
|
|
[rank3]: return _load_from_local_path(dataset_config, load_dataset_kwargs)
|
|
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/shared.py", line 247, in _load_from_local_path
|
|
[rank3]: return load_dataset(
|
|
[rank3]: ^^^^^^^^^^^^^
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/datasets/load.py", line 1508, in load_dataset
|
|
[rank3]: builder_instance.download_and_prepare(
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/datasets/builder.py", line 884, in download_and_prepare
|
|
[rank3]: self._download_and_prepare(
|
|
[rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/datasets/builder.py", line 949, in _download_and_prepare
|
|
[rank3]: raise OSError("Cannot find data file. " + "\nOriginal error:\n" + str(e)) from None
|
|
[rank3]: OSError: Cannot find data file.
|
|
[rank3]: Original error:
|
|
[rank3]: [Errno 2] No such file or directory: '/e/data1/datasets/playground/ot-baf/hf_hub/datasets/json/default-b49d7fb735d1a400/0.0.0/ff8895240c1c6d49c1ae5a8af7cd4b065d168a612f3f6bd42470270b78a42c18.incomplete/json-train-00000-00000-of-NNNNN.arrow'
|
|
|
|
[rank2]: Traceback (most recent call last):
|
|
[rank2]: File "<frozen runpy>", line 198, in _run_module_as_main
|
|
[rank2]: File "<frozen runpy>", line 88, in _run_code
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 124, in <module>
|
|
[rank2]: fire.Fire(do_cli)
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 135, in Fire
|
|
[rank2]: component_trace = _Fire(component, args, parsed_flag_args, context, name)
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 468, in _Fire
|
|
[rank2]: component, remaining_args = _CallAndUpdateTrace(
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace
|
|
[rank2]: component = fn(*varargs, **kwargs)
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 91, in do_cli
|
|
[rank2]: do_train(parsed_cfg, parsed_cli_args)
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 43, in do_train
|
|
[rank2]: dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/telemetry/errors.py", line 124, in wrapper
|
|
[rank2]: return func(*args, **kwargs)
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/common/datasets.py", line 61, in load_datasets
|
|
[rank2]: train_dataset, eval_dataset, total_num_steps, prompters = prepare_datasets(
|
|
[rank2]: ^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/utils.py", line 50, in wrapper
|
|
[rank2]: return func(*args, **kwargs)
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 65, in prepare_datasets
|
|
[rank2]: return _prepare_standard_dataset(cfg, tokenizer, processor)
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 98, in _prepare_standard_dataset
|
|
[rank2]: train_dataset, eval_dataset, prompters = loader.load(_load_datasets)
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/lock.py", line 38, in load
|
|
[rank2]: result = load_fn()
|
|
[rank2]: ^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 77, in _load_datasets
|
|
[rank2]: train_dataset, eval_dataset, prompters = _load_and_prepare_datasets(
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 496, in _load_and_prepare_datasets
|
|
[rank2]: dataset, prompters = _load_tokenized_prepared_datasets(
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 299, in _load_tokenized_prepared_datasets
|
|
[rank2]: dataset, prompters = _load_raw_datasets(
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 331, in _load_raw_datasets
|
|
[rank2]: dataset_wrapper, dataset_prompter = _load_and_process_single_dataset(
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/sft.py", line 384, in _load_and_process_single_dataset
|
|
[rank2]: dataset = load_dataset_with_config(
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/shared.py", line 118, in load_dataset_with_config
|
|
[rank2]: return _load_from_local_path(dataset_config, load_dataset_kwargs)
|
|
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/utils/data/shared.py", line 247, in _load_from_local_path
|
|
[rank2]: return load_dataset(
|
|
[rank2]: ^^^^^^^^^^^^^
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/datasets/load.py", line 1508, in load_dataset
|
|
[rank2]: builder_instance.download_and_prepare(
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/datasets/builder.py", line 884, in download_and_prepare
|
|
[rank2]: self._download_and_prepare(
|
|
[rank2]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/datasets/builder.py", line 949, in _download_and_prepare
|
|
[rank2]: raise OSError("Cannot find data file. " + "\nOriginal error:\n" + str(e)) from None
|
|
[rank2]: OSError: Cannot find data file.
|
|
[rank2]: Original error:
|
|
[rank2]: [Errno 2] No such file or directory: '/e/data1/datasets/playground/ot-baf/hf_hub/datasets/json/default-b49d7fb735d1a400/0.0.0/ff8895240c1c6d49c1ae5a8af7cd4b065d168a612f3f6bd42470270b78a42c18.incomplete/json-train-00000-00000-of-NNNNN.arrow'
|
|
[2026-04-24 20:51:05,311] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:293514] Loading dataset: /e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl with base_type: chat_template and prompt_style: None
|
|
[2026-04-24 20:51:05,316] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:389100] Loading dataset: /e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl with base_type: chat_template and prompt_style: None
|
|
[2026-04-24 20:51:05,343] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:389101] Loading raw datasets...
|
|
[2026-04-24 20:51:05,375] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:389101] Loading dataset: /e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--Sera-4.6-Lite-T2-v4-1000/snapshots/310c2661cea97bd8eb283374416193b64733fffb/sera-4.6-lite-t2_v4_1000.jsonl with base_type: chat_template and prompt_style: None
|