[2026-04-25 04:35:06,056] [DEBUG] [axolotl.utils.config.resolve_dtype:74] [PID:1706109] bf16 support detected, enabling for this configuration. [2026-04-25 04:35:06,066] [DEBUG] [axolotl.utils.config.resolve_dtype:74] [PID:971489] bf16 support detected, enabling for this configuration. [2026-04-25 04:35:06,085] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:971489] baseline 0.000GB () [2026-04-25 04:35:06,085] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:1706109] baseline 0.000GB () [2026-04-25 04:35:06,085] [INFO] [axolotl.cli.config.load_cfg:341] [PID:971489] config: { "activation_offloading": true, "adam_beta1": 0.9, "adam_beta2": 0.95, "axolotl_config_path": "/e/scratch/jureap59/feuer1/code/axolotl_configs/qwen3_8b_cf_v8_1000.yaml", "base_model": "Qwen/Qwen3-8B", "base_model_config": "Qwen/Qwen3-8B", "batch_size": 32, "bf16": true, "capabilities": { "bf16": true, "compute_capability": "sm_90", "fp8": true, "n_gpu": 4, "n_node": 1, "tf32": true }, "chat_template": "tokenizer_default", "context_parallel_size": 1, "dataloader_num_workers": 4, "dataloader_pin_memory": true, "dataloader_prefetch_factor": 256, "dataset_num_proc": 288, "dataset_prepared_path": "/e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/cf-v8-1000", "datasets": [ { "chat_template": "tokenizer_default", "data_files": [ "coderforge-preview_v6_316.jsonl" ], "ds_type": "json", "field_messages": "messages", "message_field_training": "train", "message_property_mappings": { "content": "content", "role": "role" }, "path": "/e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--CoderForge-Preview-v6-1000/snapshots/ed4a6b7b87753bc5ee6c858670ad5be422c683a9/coderforge-preview_v6_1000.jsonl", "trust_remote_code": false, "type": "chat_template" } ], "ddp": true, "deepspeed": { "bf16": { "enabled": true }, "gradient_accumulation_steps": "auto", "gradient_clipping": "auto", "train_batch_size": "auto", "train_micro_batch_size_per_gpu": "auto", "wall_clock_breakdown": false, "zero_optimization": { "contiguous_gradients": true, "gather_16bit_weights_on_model_save": true, "max_live_parameters": 0, "max_reuse_distance": 0, "overlap_comm": true, "reduce_bucket_size": "auto", "stage": 3, "stage3_param_persistence_threshold": "auto", "stage3_prefetch_bucket_size": "auto", "sub_group_size": 0 } }, "device": "cuda:0", "device_map": { "": 0 }, "dion_rank_fraction": 1.0, "dion_rank_multiple_of": 1, "eaft_alpha": 1.0, "eaft_k": 20, "env_capabilities": { "torch_version": "2.9.1" }, "eval_batch_size": 1, "eval_causal_lm_metrics": [ "sacrebleu", "comet", "ter", "chrf" ], "eval_max_new_tokens": 128, "eval_table_size": 0, "evals_per_epoch": 0, "experimental_skip_move_to_device": true, "flash_attention": true, "fp16": false, "generate_samples": false, "generation_do_sample": true, "generation_max_new_tokens": 50, "generation_prompt_ratio": 0.5, "generation_temperature": 0.7, "gradient_accumulation_steps": 8, "gradient_checkpointing": true, "gradient_checkpointing_kwargs": { "use_reentrant": true }, "include_tkps": true, "layer_offloading": false, "learning_rate": 1e-05, "lisa_layers_attribute": "model.layers", "load_best_model_at_end": false, "load_in_4bit": false, "load_in_8bit": false, "local_rank": 0, "logging_steps": 1, "lora_dropout": 0.0, "loraplus_lr_embedding": 1e-06, "loss_watchdog_patience": 3, "loss_watchdog_threshold": 5.0, "lr_scheduler": "cosine", "max_grad_norm": 1.0, "mean_resizing_embeddings": false, "merge_method": "memory_efficient", "micro_batch_size": 1, "model_config_type": "qwen3", "num_epochs": 12.0, "num_generation_samples": 3, "optimizer": "adamw_torch", "otel_metrics_host": "localhost", "otel_metrics_port": 8000, "output_dir": "/e/data1/datasets/playground/ot-baf/checkpoints/cf-v8-1000-axolotl__Qwen3-8B", "pretrain_multipack_attn": true, "profiler_steps_start": 0, "qlora_sharded_model_loading": false, "quantize_moe_experts": false, "ray_num_workers": 1, "resources_per_worker": { "GPU": 1 }, "sample_packing_bin_size": 200, "sample_packing_group_size": 100000, "save_only_model": false, "save_safetensors": true, "save_strategy": "epoch", "sequence_len": 32768, "shuffle_before_merging_datasets": false, "shuffle_merged_datasets": true, "skip_prepare_dataset": false, "streaming_multipack_buffer_size": 10000, "strict": false, "tensor_parallel_size": 1, "tf32": false, "tiled_mlp_use_original_mlp": true, "tokenizer_config": "Qwen/Qwen3-8B", "tokenizer_save_jinja_files": true, "torch_dtype": "torch.bfloat16", "train_on_inputs": false, "trl": { "async_prefetch": false, "log_completions": false, "mask_truncated_completions": false, "ref_model_mixup_alpha": 0.9, "ref_model_sync_steps": 64, "replay_buffer_size": 0, "replay_recompute_logps": true, "reroll_max_groups": 1, "reroll_start_fraction": 1.0, "reward_num_workers": 1, "scale_rewards": true, "skip_zero_advantage_batches": true, "sync_ref_model": false, "use_data_producer": false, "use_vllm": false, "vllm_lora_sync": false, "vllm_server_host": "0.0.0.0", "vllm_server_port": 8000 }, "use_otel_metrics": false, "use_ray": false, "val_set_size": 0.0, "vllm": { "device": "auto", "dtype": "auto", "gpu_memory_utilization": 0.9, "host": "0.0.0.0", "port": 8000 }, "wandb_name": "cf-v8-1000-axolotl__Qwen3-8B", "warmup_ratio": 0.1875, "weight_decay": 0.01, "world_size": 4 } [2026-04-25 04:35:06,087] [INFO] [axolotl.cli.config.load_cfg:341] [PID:1706109] config: { "activation_offloading": true, "adam_beta1": 0.9, "adam_beta2": 0.95, "axolotl_config_path": "/e/scratch/jureap59/feuer1/code/axolotl_configs/qwen3_8b_cf_v8_1000.yaml", "base_model": "Qwen/Qwen3-8B", "base_model_config": "Qwen/Qwen3-8B", "batch_size": 32, "bf16": true, "capabilities": { "bf16": true, "compute_capability": "sm_90", "fp8": true, "n_gpu": 4, "n_node": 1, "tf32": true }, "chat_template": "tokenizer_default", "context_parallel_size": 1, "dataloader_num_workers": 4, "dataloader_pin_memory": true, "dataloader_prefetch_factor": 256, "dataset_num_proc": 288, "dataset_prepared_path": "/e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/cf-v8-1000", "datasets": [ { "chat_template": "tokenizer_default", "data_files": [ "coderforge-preview_v6_316.jsonl" ], "ds_type": "json", "field_messages": "messages", "message_field_training": "train", "message_property_mappings": { "content": "content", "role": "role" }, "path": "/e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--CoderForge-Preview-v6-1000/snapshots/ed4a6b7b87753bc5ee6c858670ad5be422c683a9/coderforge-preview_v6_1000.jsonl", "trust_remote_code": false, "type": "chat_template" } ], "ddp": true, "deepspeed": { "bf16": { "enabled": true }, "gradient_accumulation_steps": "auto", "gradient_clipping": "auto", "train_batch_size": "auto", "train_micro_batch_size_per_gpu": "auto", "wall_clock_breakdown": false, "zero_optimization": { "contiguous_gradients": true, "gather_16bit_weights_on_model_save": true, "max_live_parameters": 0, "max_reuse_distance": 0, "overlap_comm": true, "reduce_bucket_size": "auto", "stage": 3, "stage3_param_persistence_threshold": "auto", "stage3_prefetch_bucket_size": "auto", "sub_group_size": 0 } }, "device": "cuda:0", "device_map": { "": 0 }, "dion_rank_fraction": 1.0, "dion_rank_multiple_of": 1, "eaft_alpha": 1.0, "eaft_k": 20, "env_capabilities": { "torch_version": "2.9.1" }, "eval_batch_size": 1, "eval_causal_lm_metrics": [ "sacrebleu", "comet", "ter", "chrf" ], "eval_max_new_tokens": 128, "eval_table_size": 0, "evals_per_epoch": 0, "experimental_skip_move_to_device": true, "flash_attention": true, "fp16": false, "generate_samples": false, "generation_do_sample": true, "generation_max_new_tokens": 50, "generation_prompt_ratio": 0.5, "generation_temperature": 0.7, "gradient_accumulation_steps": 8, "gradient_checkpointing": true, "gradient_checkpointing_kwargs": { "use_reentrant": true }, "include_tkps": true, "layer_offloading": false, "learning_rate": 1e-05, "lisa_layers_attribute": "model.layers", "load_best_model_at_end": false, "load_in_4bit": false, "load_in_8bit": false, "local_rank": 0, "logging_steps": 1, "lora_dropout": 0.0, "loraplus_lr_embedding": 1e-06, "loss_watchdog_patience": 3, "loss_watchdog_threshold": 5.0, "lr_scheduler": "cosine", "max_grad_norm": 1.0, "mean_resizing_embeddings": false, "merge_method": "memory_efficient", "micro_batch_size": 1, "model_config_type": "qwen3", "num_epochs": 12.0, "num_generation_samples": 3, "optimizer": "adamw_torch", "otel_metrics_host": "localhost", "otel_metrics_port": 8000, "output_dir": "/e/data1/datasets/playground/ot-baf/checkpoints/cf-v8-1000-axolotl__Qwen3-8B", "pretrain_multipack_attn": true, "profiler_steps_start": 0, "qlora_sharded_model_loading": false, "quantize_moe_experts": false, "ray_num_workers": 1, "resources_per_worker": { "GPU": 1 }, "sample_packing_bin_size": 200, "sample_packing_group_size": 100000, "save_only_model": false, "save_safetensors": true, "save_strategy": "epoch", "sequence_len": 32768, "shuffle_before_merging_datasets": false, "shuffle_merged_datasets": true, "skip_prepare_dataset": false, "streaming_multipack_buffer_size": 10000, "strict": false, "tensor_parallel_size": 1, "tf32": false, "tiled_mlp_use_original_mlp": true, "tokenizer_config": "Qwen/Qwen3-8B", "tokenizer_save_jinja_files": true, "torch_dtype": "torch.bfloat16", "train_on_inputs": false, "trl": { "async_prefetch": false, "log_completions": false, "mask_truncated_completions": false, "ref_model_mixup_alpha": 0.9, "ref_model_sync_steps": 64, "replay_buffer_size": 0, "replay_recompute_logps": true, "reroll_max_groups": 1, "reroll_start_fraction": 1.0, "reward_num_workers": 1, "scale_rewards": true, "skip_zero_advantage_batches": true, "sync_ref_model": false, "use_data_producer": false, "use_vllm": false, "vllm_lora_sync": false, "vllm_server_host": "0.0.0.0", "vllm_server_port": 8000 }, "use_otel_metrics": false, "use_ray": false, "val_set_size": 0.0, "vllm": { "device": "auto", "dtype": "auto", "gpu_memory_utilization": 0.9, "host": "0.0.0.0", "port": 8000 }, "wandb_name": "cf-v8-1000-axolotl__Qwen3-8B", "warmup_ratio": 0.1875, "weight_decay": 0.01, "world_size": 4 } [2026-04-25 04:35:06,120] [INFO] [axolotl.cli.checks.check_user_token:37] [PID:971489] Skipping HuggingFace token verification because HF_HUB_OFFLINE is set to True. Only local files will be used. [2026-04-25 04:35:06,126] [INFO] [axolotl.cli.checks.check_user_token:37] [PID:1706109] Skipping HuggingFace token verification because HF_HUB_OFFLINE is set to True. Only local files will be used. [2026-04-25 04:35:06,195] [DEBUG] [axolotl.utils.config.resolve_dtype:74] [PID:1998124] bf16 support detected, enabling for this configuration. [2026-04-25 04:35:06,207] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:1998124] baseline 0.000GB () [2026-04-25 04:35:06,207] [INFO] [axolotl.cli.config.load_cfg:341] [PID:1998124] config: { "activation_offloading": true, "adam_beta1": 0.9, "adam_beta2": 0.95, "axolotl_config_path": "/e/scratch/jureap59/feuer1/code/axolotl_configs/qwen3_8b_cf_v8_1000.yaml", "base_model": "Qwen/Qwen3-8B", "base_model_config": "Qwen/Qwen3-8B", "batch_size": 32, "bf16": true, "capabilities": { "bf16": true, "compute_capability": "sm_90", "fp8": true, "n_gpu": 4, "n_node": 1, "tf32": true }, "chat_template": "tokenizer_default", "context_parallel_size": 1, "dataloader_num_workers": 4, "dataloader_pin_memory": true, "dataloader_prefetch_factor": 256, "dataset_num_proc": 288, "dataset_prepared_path": "/e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/cf-v8-1000", "datasets": [ { "chat_template": "tokenizer_default", "data_files": [ "coderforge-preview_v6_316.jsonl" ], "ds_type": "json", "field_messages": "messages", "message_field_training": "train", "message_property_mappings": { "content": "content", "role": "role" }, "path": "/e/data1/datasets/playground/ot-baf/hf_hub/datasets--laion--CoderForge-Preview-v6-1000/snapshots/ed4a6b7b87753bc5ee6c858670ad5be422c683a9/coderforge-preview_v6_1000.jsonl", "trust_remote_code": false, "type": "chat_template" } ], "ddp": true, "deepspeed": { "bf16": { "enabled": true }, "gradient_accumulation_steps": "auto", "gradient_clipping": "auto", "train_batch_size": "auto", "train_micro_batch_size_per_gpu": "auto", "wall_clock_breakdown": false, "zero_optimization": { "contiguous_gradients": true, "gather_16bit_weights_on_model_save": true, "max_live_parameters": 0, "max_reuse_distance": 0, "overlap_comm": true, "reduce_bucket_size": "auto", "stage": 3, "stage3_param_persistence_threshold": "auto", "stage3_prefetch_bucket_size": "auto", "sub_group_size": 0 } }, "device": "cuda:0", "device_map": { "": 0 }, "dion_rank_fraction": 1.0, "dion_rank_multiple_of": 1, "eaft_alpha": 1.0, "eaft_k": 20, "env_capabilities": { "torch_version": "2.9.1" }, "eval_batch_size": 1, "eval_causal_lm_metrics": [ "sacrebleu", "comet", "ter", "chrf" ], "eval_max_new_tokens": 128, "eval_table_size": 0, "evals_per_epoch": 0, "experimental_skip_move_to_device": true, "flash_attention": true, "fp16": false, "generate_samples": false, "generation_do_sample": true, "generation_max_new_tokens": 50, "generation_prompt_ratio": 0.5, "generation_temperature": 0.7, "gradient_accumulation_steps": 8, "gradient_checkpointing": true, "gradient_checkpointing_kwargs": { "use_reentrant": true }, "include_tkps": true, "layer_offloading": false, "learning_rate": 1e-05, "lisa_layers_attribute": "model.layers", "load_best_model_at_end": false, "load_in_4bit": false, "load_in_8bit": false, "local_rank": 0, "logging_steps": 1, "lora_dropout": 0.0, "loraplus_lr_embedding": 1e-06, "loss_watchdog_patience": 3, "loss_watchdog_threshold": 5.0, "lr_scheduler": "cosine", "max_grad_norm": 1.0, "mean_resizing_embeddings": false, "merge_method": "memory_efficient", "micro_batch_size": 1, "model_config_type": "qwen3", "num_epochs": 12.0, "num_generation_samples": 3, "optimizer": "adamw_torch", "otel_metrics_host": "localhost", "otel_metrics_port": 8000, "output_dir": "/e/data1/datasets/playground/ot-baf/checkpoints/cf-v8-1000-axolotl__Qwen3-8B", "pretrain_multipack_attn": true, "profiler_steps_start": 0, "qlora_sharded_model_loading": false, "quantize_moe_experts": false, "ray_num_workers": 1, "resources_per_worker": { "GPU": 1 }, "sample_packing_bin_size": 200, "sample_packing_group_size": 100000, "save_only_model": false, "save_safetensors": true, "save_strategy": "epoch", "sequence_len": 32768, "shuffle_before_merging_datasets": false, "shuffle_merged_datasets": true, "skip_prepare_dataset": false, "streaming_multipack_buffer_size": 10000, "strict": false, "tensor_parallel_size": 1, "tf32": false, "tiled_mlp_use_original_mlp": true, "tokenizer_config": "Qwen/Qwen3-8B", "tokenizer_save_jinja_files": true, "torch_dtype": "torch.bfloat16", "train_on_inputs": false, "trl": { "async_prefetch": false, "log_completions": false, "mask_truncated_completions": false, "ref_model_mixup_alpha": 0.9, "ref_model_sync_steps": 64, "replay_buffer_size": 0, "replay_recompute_logps": true, "reroll_max_groups": 1, "reroll_start_fraction": 1.0, "reward_num_workers": 1, "scale_rewards": true, "skip_zero_advantage_batches": true, "sync_ref_model": false, "use_data_producer": false, "use_vllm": false, "vllm_lora_sync": false, "vllm_server_host": "0.0.0.0", "vllm_server_port": 8000 }, "use_otel_metrics": false, "use_ray": false, "val_set_size": 0.0, "vllm": { "device": "auto", "dtype": "auto", "gpu_memory_utilization": 0.9, "host": "0.0.0.0", "port": 8000 }, "wandb_name": "cf-v8-1000-axolotl__Qwen3-8B", "warmup_ratio": 0.1875, "weight_decay": 0.01, "world_size": 4 } [2026-04-25 04:35:06,209] [INFO] [axolotl.cli.checks.check_user_token:37] [PID:1998124] Skipping HuggingFace token verification because HF_HUB_OFFLINE is set to True. Only local files will be used. [2026-04-25 04:35:06,598] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:1706109] EOS: 151645 / <|im_end|> [2026-04-25 04:35:06,600] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:1706109] BOS: None / None [2026-04-25 04:35:06,600] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:1706109] PAD: 151643 / <|endoftext|> [2026-04-25 04:35:06,601] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:1706109] UNK: None / None [2026-04-25 04:35:06,603] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:971489] EOS: 151645 / <|im_end|> [2026-04-25 04:35:06,605] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:971489] BOS: None / None [2026-04-25 04:35:06,605] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:971489] PAD: 151643 / <|endoftext|> [2026-04-25 04:35:06,605] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:971489] UNK: None / None [2026-04-25 04:35:06,655] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:475] [PID:1706109] Loading prepared dataset from disk at /e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/cf-v8-1000/db17953b8a5c66f4f9e999ac345b71b1... [2026-04-25 04:35:06,659] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:475] [PID:971489] Loading prepared dataset from disk at /e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/cf-v8-1000/db17953b8a5c66f4f9e999ac345b71b1... [2026-04-25 04:35:06,709] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:1998124] EOS: 151645 / <|im_end|> [2026-04-25 04:35:06,711] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:1998124] BOS: None / None [2026-04-25 04:35:06,711] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:1998124] PAD: 151643 / <|endoftext|> [2026-04-25 04:35:06,711] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:1998124] UNK: None / None [2026-04-25 04:35:06,733] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:1706109] total_num_tokens: 7_117_278 [2026-04-25 04:35:06,744] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:971489] total_num_tokens: 7_117_278 [2026-04-25 04:35:06,889] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:971489] `total_supervised_tokens: 1_724_858` [2026-04-25 04:35:06,889] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:971489] total_num_steps: 98 [2026-04-25 04:35:06,890] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:971489] Maximum number of steps set at 98 [2026-04-25 04:35:06,889] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:1706109] `total_supervised_tokens: 1_724_858` [2026-04-25 04:35:06,891] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:1706109] total_num_steps: 98 [2026-04-25 04:35:06,891] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:1706109] Maximum number of steps set at 98 [2026-04-25 04:35:06,897] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:475] [PID:1998124] Loading prepared dataset from disk at /e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/cf-v8-1000/db17953b8a5c66f4f9e999ac345b71b1... [2026-04-25 04:35:06,915] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:1998124] total_num_tokens: 7_117_278 [2026-04-25 04:35:06,926] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:1998124] `total_supervised_tokens: 1_724_858` [2026-04-25 04:35:06,926] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:1998124] total_num_steps: 98 [2026-04-25 04:35:06,926] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:1998124] Maximum number of steps set at 98 [2026-04-25 04:35:06,926] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:1706109] loading tokenizer... Qwen/Qwen3-8B [2026-04-25 04:35:06,927] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:971489] loading tokenizer... Qwen/Qwen3-8B [2026-04-25 04:35:06,964] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:1998124] loading tokenizer... Qwen/Qwen3-8B [2026-04-25 04:35:07,178] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:1706109] EOS: 151645 / <|im_end|> [2026-04-25 04:35:07,181] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:1706109] BOS: None / None [2026-04-25 04:35:07,181] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:1706109] PAD: 151643 / <|endoftext|> [2026-04-25 04:35:07,181] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:1706109] UNK: None / None [2026-04-25 04:35:07,181] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:1706109] Loading model [2026-04-25 04:35:07,191] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:971489] EOS: 151645 / <|im_end|> [2026-04-25 04:35:07,193] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:971489] BOS: None / None [2026-04-25 04:35:07,193] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:971489] PAD: 151643 / <|endoftext|> [2026-04-25 04:35:07,193] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:971489] UNK: None / None [2026-04-25 04:35:07,193] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:971489] Loading model [2026-04-25 04:35:07,225] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:1998124] EOS: 151645 / <|im_end|> [2026-04-25 04:35:07,228] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:1998124] BOS: None / None [2026-04-25 04:35:07,228] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:1998124] PAD: 151643 / <|endoftext|> [2026-04-25 04:35:07,228] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:1998124] UNK: None / None [2026-04-25 04:35:07,228] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:1998124] Loading model [2026-04-25 04:35:07,250] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:1706109] Patched Trainer.evaluation_loop with nanmean loss calculation [2026-04-25 04:35:07,250] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:971489] Patched Trainer.evaluation_loop with nanmean loss calculation [2026-04-25 04:35:07,251] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:1998124] Patched Trainer.evaluation_loop with nanmean loss calculation [2026-04-25 04:35:07,253] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:1706109] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation [2026-04-25 04:35:07,255] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:971489] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation [2026-04-25 04:35:07,256] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:1998124] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation [2026-04-25 04:35:07,457] [INFO] [axolotl.monkeypatch.attention.flash_attn_4.patch_flash_attn_4:52] [PID:971489] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 [2026-04-25 04:35:07,459] [INFO] [axolotl.monkeypatch.deepspeed_utils.patch_checkpoint_wrapper_setattr:52] [PID:971489] CheckpointWrapper patched to forward DeepSpeed attributes [2026-04-25 04:35:07,621] [INFO] [axolotl.monkeypatch.attention.flash_attn_4.patch_flash_attn_4:52] [PID:1998124] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 [2026-04-25 04:35:07,635] [INFO] [axolotl.monkeypatch.deepspeed_utils.patch_checkpoint_wrapper_setattr:52] [PID:1998124] CheckpointWrapper patched to forward DeepSpeed attributes [2026-04-25 04:35:07,663] [INFO] [axolotl.monkeypatch.attention.flash_attn_4.patch_flash_attn_4:52] [PID:1706109] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 [2026-04-25 04:35:07,665] [INFO] [axolotl.monkeypatch.deepspeed_utils.patch_checkpoint_wrapper_setattr:52] [PID:1706109] CheckpointWrapper patched to forward DeepSpeed attributes [2026-04-25 04:35:33,798] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:361] [PID:1706109] Converting modules to torch.bfloat16 [2026-04-25 04:35:33,800] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:361] [PID:971489] Converting modules to torch.bfloat16 [2026-04-25 04:35:33,801] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:1706109] Memory usage after model load 4.973GB (+4.973GB allocated, +5.080GB reserved) [2026-04-25 04:35:33,806] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:971489] Memory usage after model load 4.973GB (+4.973GB allocated, +5.080GB reserved) [2026-04-25 04:35:33,980] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:361] [PID:1998124] Converting modules to torch.bfloat16 [2026-04-25 04:35:33,985] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:1998124] Memory usage after model load 4.973GB (+4.973GB allocated, +5.080GB reserved) [2026-04-25 04:35:40,282] [INFO] [axolotl.train.save_initial_configs:421] [PID:1706109] Pre-saving tokenizer to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v8-1000-axolotl__Qwen3-8B... [2026-04-25 04:35:40,297] [INFO] [axolotl.train.save_initial_configs:421] [PID:1998124] Pre-saving tokenizer to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v8-1000-axolotl__Qwen3-8B... [2026-04-25 04:35:40,298] [INFO] [axolotl.train.save_initial_configs:421] [PID:971489] Pre-saving tokenizer to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v8-1000-axolotl__Qwen3-8B... [2026-04-25 04:35:40,388] [INFO] [axolotl.train.save_initial_configs:426] [PID:1706109] Pre-saving model config to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v8-1000-axolotl__Qwen3-8B... [2026-04-25 04:35:40,395] [INFO] [axolotl.train.execute_training:222] [PID:1706109] Starting trainer... [2026-04-25 04:35:40,471] [INFO] [axolotl.train.save_initial_configs:426] [PID:1998124] Pre-saving model config to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v8-1000-axolotl__Qwen3-8B... [2026-04-25 04:35:40,476] [INFO] [axolotl.train.execute_training:222] [PID:1998124] Starting trainer... [2026-04-25 04:35:40,509] [INFO] [axolotl.train.save_initial_configs:426] [PID:971489] Pre-saving model config to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v8-1000-axolotl__Qwen3-8B... [2026-04-25 04:35:40,519] [INFO] [axolotl.train.execute_training:222] [PID:971489] Starting trainer... [2026-04-25 04:35:40,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,786] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,798] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,864] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,882] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,910] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,941] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,947] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,956] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,975] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:40,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,039] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:971489] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-25 04:35:41,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_id to wrapped module Qwen3DecoderLayer 0%| | 0/98 [00:00", line 198, in _run_module_as_main [rank3]: File "", line 88, in _run_code [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 124, in [rank3]: fire.Fire(do_cli) [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 135, in Fire [rank3]: component_trace = _Fire(component, args, parsed_flag_args, context, name) [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 468, in _Fire [rank3]: component, remaining_args = _CallAndUpdateTrace( [rank3]: ^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/fire/core.py", line 684, in _CallAndUpdateTrace [rank3]: component = fn(*varargs, **kwargs) [rank3]: ^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 91, in do_cli [rank3]: do_train(parsed_cfg, parsed_cli_args) [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/cli/train.py", line 45, in do_train [rank3]: model, tokenizer, trainer = train(cfg=cfg, dataset_meta=dataset_meta) [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/telemetry/errors.py", line 124, in wrapper [rank3]: return func(*args, **kwargs) [rank3]: ^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/train.py", line 595, in train [rank3]: execute_training(cfg, trainer, resume_from_checkpoint) [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/train.py", line 223, in execute_training [rank3]: trainer.train(resume_from_checkpoint=resume_from_checkpoint) [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/transformers/trainer.py", line 1424, in train [rank3]: return inner_training_loop( [rank3]: ^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/transformers/trainer.py", line 1506, in _inner_training_loop [rank3]: self._run_epoch( [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/transformers/trainer.py", line 1734, in _run_epoch [rank3]: tr_loss_step = self.training_step(model, inputs, num_items_in_batch) [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/core/trainers/mixins/layer_offloading.py", line 304, in training_step [rank3]: return super().training_step(*args, **kwargs) [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/code/axolotl/src/axolotl/core/trainers/mixins/activation_checkpointing.py", line 46, in training_step [rank3]: return super().training_step(*args, **kwargs) [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/transformers/trainer.py", line 1934, in training_step [rank3]: self.accelerator.backward(loss, **kwargs) [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/accelerate/accelerator.py", line 2830, in backward [rank3]: self.deepspeed_engine_wrapped.backward(loss, sync_gradients=self.sync_gradients, **kwargs) [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/accelerate/utils/deepspeed.py", line 270, in backward [rank3]: self.engine.backward(loss, **kwargs) [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/deepspeed/utils/nvtx.py", line 20, in wrapped_fn [rank3]: ret_val = func(*args, **kwargs) [rank3]: ^^^^^^^^^^^^^^^^^^^^^ [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/deepspeed/runtime/engine.py", line 2625, in backward [rank3]: loss.backward(**backward_kwargs) [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/torch/_tensor.py", line 625, in backward [rank3]: torch.autograd.backward( [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/torch/autograd/__init__.py", line 354, in backward [rank3]: _engine_run_backward( [rank3]: File "/e/scratch/jureap59/feuer1/miniforge3/envs/sera-axolotl/lib/python3.12/site-packages/torch/autograd/graph.py", line 841, in _engine_run_backward [rank3]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass [rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank3]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 18.00 GiB. GPU 3 has a total capacity of 95.00 GiB of which 21.15 GiB is free. Including non-PyTorch memory, this process has 73.42 GiB memory in use. Of the allocated memory 71.25 GiB is allocated by PyTorch, and 116.02 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) [2026-04-25 04:37:26,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:26,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,477] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,702] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,882] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:27,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:28,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,910] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:38,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:39,973] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,039] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,171] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,171] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:40,706] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:50,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:50,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:50,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:50,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:50,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:50,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:50,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:50,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:50,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,539] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:51,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,363] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:37:52,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:00,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:00,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:00,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:00,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:00,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:00,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:00,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,552] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:01,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,162] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,748] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:02,975] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:03,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:03,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:10,600] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:10,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:10,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:10,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:10,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:10,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:10,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:10,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:10,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,692] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:11,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,079] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,138] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,514] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:12,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:13,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:13,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:13,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:13,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:22,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:22,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,882] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:23,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,612] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:24,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:25,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:25,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:25,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:25,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:25,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:25,402] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:25,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:25,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:25,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:34,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:34,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:34,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:34,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:34,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:34,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:34,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,040] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,665] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,703] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:35,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,432] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,678] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:36,975] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:37,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:37,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:37,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:37,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:37,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:37,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:37,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:37,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,287] [WARNING] [stage3.py:2497:step] 8 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 2%|▏ | 2/98 [03:04<2:27:16, 92.05s/it] {'loss': '0.6753', 'grad_norm': '18.13', 'learning_rate': '5.556e-07', 'ppl': '1.965', 'memory/max_active (GiB)': '91.47', 'memory/max_allocated (GiB)': '91.47', 'memory/device_reserved (GiB)': '92.81', 'tokens/train_per_sec_per_gpu': '70.95', 'tokens/total': 1736832, 'tokens/trainable': 410733, 'epoch': '0.2424'} 2%|▏ | 2/98 [03:04<2:27:16, 92.05s/it][2026-04-25 04:38:46,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,596] [WARNING] [stage3.py:2497:step] 8 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 2%|▏ | 2/98 [03:04<2:27:47, 92.37s/it] {'loss': '0.6753', 'grad_norm': '18.13', 'learning_rate': '5.556e-07', 'ppl': '1.965', 'memory/max_active (GiB)': '91.47', 'memory/max_allocated (GiB)': '91.47', 'memory/device_reserved (GiB)': '92.81', 'tokens/train_per_sec_per_gpu': '70.61', 'tokens/total': 1736832, 'tokens/trainable': 410733, 'epoch': '0.2424'} 2%|▏ | 2/98 [03:04<2:27:47, 92.37s/it][2026-04-25 04:38:46,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,701] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:46,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,947] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:47,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,138] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:48,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:49,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,362] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:58,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,301] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,703] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:38:59,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,171] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:00,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:12,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:13,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:14,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:25,830] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:25,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:25,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:25,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:25,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:25,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:25,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:25,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,612] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,634] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,830] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,852] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:26,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:27,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,791] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,947] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:36,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,432] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,843] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:37,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:38,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,347] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,614] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,956] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:47,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,039] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,138] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:48,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:49,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,539] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:39:59,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,853] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:00,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,251] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:01,958] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:02,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:10,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:10,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:10,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:10,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:10,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:10,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:10,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:10,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:10,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:10,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:10,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:10,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,748] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:11,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,322] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,536] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,798] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,852] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:12,973] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:13,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:13,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:13,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:13,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:13,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:13,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:13,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:13,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:13,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:13,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:13,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:22,683] [WARNING] [stage3.py:2497:step] 12 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 3%|▎ | 3/98 [04:40<2:28:52, 94.03s/it] {'loss': '0.716', 'grad_norm': '18.98', 'learning_rate': '1.111e-06', 'ppl': '2.046', 'memory/max_active (GiB)': '91.23', 'memory/max_allocated (GiB)': '91.23', 'memory/device_reserved (GiB)': '92.87', 'tokens/train_per_sec_per_gpu': '54.67', 'tokens/total': 2630912, 'tokens/trainable': 610856, 'epoch': '0.3636'} 3%|▎ | 3/98 [04:40<2:28:52, 94.03s/it][2026-04-25 04:40:22,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:22,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:22,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:22,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:22,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:22,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,623] [WARNING] [stage3.py:2497:step] 12 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time [2026-04-25 04:40:23,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer 3%|▎ | 3/98 [04:41<2:29:36, 94.49s/it] {'loss': '0.7156', 'grad_norm': '18.91', 'learning_rate': '1.111e-06', 'ppl': '2.045', 'memory/max_active (GiB)': '91.23', 'memory/max_allocated (GiB)': '91.23', 'memory/device_reserved (GiB)': '92.71', 'tokens/train_per_sec_per_gpu': '54.32', 'tokens/total': 2630912, 'tokens/trainable': 610856, 'epoch': '0.3636'} 3%|▎ | 3/98 [04:41<2:29:36, 94.49s/it][2026-04-25 04:40:23,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:23,947] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,193] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,322] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:24,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,825] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:25,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:26,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:34,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:34,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,634] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:35,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,376] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,601] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,859] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,955] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:36,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,193] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,570] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:37,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:44,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:44,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:44,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:44,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:44,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:44,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:44,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:44,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,975] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:45,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:46,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:47,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:47,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:47,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:47,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:47,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:47,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:54,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,477] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:55,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,251] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:56,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:57,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:57,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:40:57,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,600] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:07,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,347] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,600] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,678] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,825] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:08,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,480] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:09,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:10,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:10,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:10,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:10,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:10,313] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:10,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:10,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:10,507] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:10,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:18,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:18,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:18,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:18,706] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:18,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:18,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:18,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:18,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:18,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,162] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,958] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:19,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:20,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:21,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:21,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:21,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:21,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:21,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:21,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:21,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:21,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:21,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:21,511] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:21,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:28,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:28,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:28,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:28,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:28,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:28,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:28,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:28,973] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:29,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,757] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:30,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:31,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:32,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:32,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:32,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:42,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,601] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:43,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,773] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:44,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:45,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:45,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:45,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:45,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:45,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:45,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:45,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:45,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:45,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:45,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:45,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,042] [WARNING] [stage3.py:2497:step] 11 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 4%|▍ | 4/98 [06:12<2:26:16, 93.37s/it] {'loss': '0.6553', 'grad_norm': '15.84', 'learning_rate': '1.667e-06', 'ppl': '1.926', 'memory/max_active (GiB)': '91', 'memory/max_allocated (GiB)': '91', 'memory/device_reserved (GiB)': '92.86', 'tokens/train_per_sec_per_gpu': '86.33', 'tokens/total': 3517120, 'tokens/trainable': 840250, 'epoch': '0.4848'} 4%|▍ | 4/98 [06:12<2:26:16, 93.37s/it][2026-04-25 04:41:55,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:55,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,322] [WARNING] [stage3.py:2497:step] 12 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 4%|▍ | 4/98 [06:14<2:26:55, 93.79s/it] {'loss': '0.6547', 'grad_norm': '15.75', 'learning_rate': '1.667e-06', 'ppl': '1.925', 'memory/max_active (GiB)': '91', 'memory/max_allocated (GiB)': '91', 'memory/device_reserved (GiB)': '92.79', 'tokens/train_per_sec_per_gpu': '86.01', 'tokens/total': 3517120, 'tokens/trainable': 840250, 'epoch': '0.4848'} 4%|▍ | 4/98 [06:14<2:26:55, 93.79s/it][2026-04-25 04:41:56,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,362] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,973] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:56,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,171] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,600] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:57,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:58,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:58,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:58,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:58,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:58,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:58,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:58,402] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:41:58,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:06,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:06,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:06,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:06,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:06,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:06,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:06,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:06,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:06,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:06,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,851] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:07,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,734] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,941] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:08,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,600] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:09,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:10,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:10,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:10,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,347] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,852] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:20,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,319] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:21,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:22,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:22,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:22,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:22,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:22,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:22,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:22,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:22,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:31,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:31,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:31,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,480] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:32,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,325] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:33,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:34,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:34,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:34,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:34,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:34,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:34,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:34,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:34,363] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:34,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:34,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,514] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,615] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,734] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:44,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,162] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,268] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,791] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:45,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:46,829] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:53,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:53,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:53,859] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:53,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:53,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:54,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,692] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:55,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:42:56,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,312] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,864] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:05,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,498] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,614] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:06,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,297] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,791] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,853] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:07,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:08,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:08,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:08,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:08,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:08,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:08,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:08,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:08,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:08,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:08,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:17,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,362] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,612] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,734] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:18,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:19,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,703] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:20,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:29,394] [WARNING] [stage3.py:2497:step] 11 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 5%|▌ | 5/98 [07:47<2:25:16, 93.72s/it] {'loss': '0.6338', 'grad_norm': '13.54', 'learning_rate': '2.222e-06', 'ppl': '1.885', 'memory/max_active (GiB)': '90.05', 'memory/max_allocated (GiB)': '90.05', 'memory/device_reserved (GiB)': '92.71', 'tokens/train_per_sec_per_gpu': '76.74', 'tokens/total': 4406272, 'tokens/trainable': 1061463, 'epoch': '0.6061'} 5%|▌ | 5/98 [07:47<2:25:16, 93.72s/it][2026-04-25 04:43:29,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:29,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:29,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:29,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:29,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:29,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:29,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:29,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:29,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:29,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:29,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:30,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,340] [WARNING] [stage3.py:2497:step] 11 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 5%|▌ | 5/98 [07:49<2:26:03, 94.23s/it] {'loss': '0.6333', 'grad_norm': '13.49', 'learning_rate': '2.222e-06', 'ppl': '1.884', 'memory/max_active (GiB)': '90.05', 'memory/max_allocated (GiB)': '90.05', 'memory/device_reserved (GiB)': '92.47', 'tokens/train_per_sec_per_gpu': '76.2', 'tokens/total': 4406272, 'tokens/trainable': 1061463, 'epoch': '0.6061'} 5%|▌ | 5/98 [07:49<2:26:03, 94.23s/it][2026-04-25 04:43:31,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:31,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,852] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:32,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:33,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:33,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:33,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:33,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:33,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:33,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:33,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:33,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:33,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:40,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:40,552] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:40,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:40,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:40,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:40,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:40,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:40,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:40,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:40,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:40,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:40,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:41,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:42,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:42,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:42,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:42,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:42,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:42,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:42,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:42,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:42,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,347] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,853] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,947] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:43,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:44,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:44,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:44,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:44,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:44,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:44,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:44,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:49,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:49,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:49,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:49,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:49,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:49,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:49,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:49,514] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:49,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:49,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:49,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:50,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,312] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:51,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:52,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:53,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:53,079] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:53,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:53,200] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:43:53,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,757] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:00,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,206] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,910] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:01,973] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:02,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:03,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:04,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:04,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:04,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:04,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:04,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:04,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:04,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,268] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,657] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:12,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:13,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:14,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:14,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:14,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:14,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:14,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:14,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:14,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:14,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,634] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,864] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:15,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:16,040] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:16,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:16,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:16,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:16,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:16,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:16,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:16,451] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:16,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,843] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:23,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:24,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:25,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,327] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,477] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,882] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:26,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:27,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:27,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:27,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,500] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:32,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:33,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:34,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,701] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,882] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:35,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:36,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:36,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:36,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:36,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:36,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:36,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:36,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:36,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:36,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:36,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:36,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:43,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,612] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:44,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:45,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:45,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:45,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:45,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:45,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:45,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:45,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:45,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:45,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:45,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:45,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:46,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:47,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:56,407] [WARNING] [stage3.py:2497:step] 8 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 6%|▌ | 6/98 [09:14<2:20:12, 91.44s/it] {'loss': '0.5864', 'grad_norm': '8.529', 'learning_rate': '2.778e-06', 'ppl': '1.798', 'memory/max_active (GiB)': '90.53', 'memory/max_allocated (GiB)': '90.53', 'memory/device_reserved (GiB)': '92.3', 'tokens/train_per_sec_per_gpu': '83.39', 'tokens/total': 5264000, 'tokens/trainable': 1276963, 'epoch': '0.7273'} 6%|▌ | 6/98 [09:14<2:20:12, 91.44s/it][2026-04-25 04:44:56,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:56,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:56,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:56,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:56,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:56,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:56,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:56,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:56,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:56,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,362] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,600] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:57,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:58,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:58,079] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:58,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:58,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:58,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:58,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:58,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:58,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:58,500] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,278] [WARNING] [stage3.py:2497:step] 6 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 6%|▌ | 6/98 [09:17<2:21:12, 92.09s/it] {'loss': '0.5864', 'grad_norm': '8.529', 'learning_rate': '2.778e-06', 'ppl': '1.798', 'memory/max_active (GiB)': '90.05', 'memory/max_allocated (GiB)': '90.05', 'memory/device_reserved (GiB)': '91.81', 'tokens/train_per_sec_per_gpu': '82.51', 'tokens/total': 5264000, 'tokens/trainable': 1276963, 'epoch': '0.7273'} 6%|▌ | 6/98 [09:17<2:21:12, 92.09s/it][2026-04-25 04:44:59,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,862] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:44:59,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:00,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:01,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:01,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:01,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:01,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:01,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:01,301] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:01,362] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,829] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:07,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,507] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:08,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:09,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:10,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:10,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:10,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:10,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:10,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:10,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:10,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:10,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:10,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:10,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:11,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:12,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:12,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:12,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:12,171] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:12,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:12,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:12,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:17,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,872] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:18,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:19,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:20,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:20,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:20,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:20,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:20,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:20,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:20,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:21,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:22,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:22,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:22,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:22,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:22,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:22,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:22,451] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:22,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:22,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:27,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:28,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:29,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:29,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:29,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:29,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:29,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:29,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:29,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:30,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:30,600] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:30,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:30,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:30,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:30,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:30,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:30,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:31,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:32,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:32,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:32,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:32,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:32,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:32,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:32,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:32,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:32,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:32,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:32,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,830] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:39,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,327] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,595] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:40,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,754] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:43,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:44,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,512] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:47,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:48,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:49,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:49,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:49,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,657] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:51,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,319] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,862] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:52,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:53,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:53,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:53,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:53,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:53,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:45:53,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,791] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:01,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:02,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:03,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:03,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:05,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:05,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:05,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:05,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:05,692] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:05,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:05,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:05,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:05,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:05,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,703] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,757] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:06,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:07,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:07,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:07,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:07,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:07,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:07,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:07,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:10,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,138] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:11,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:12,956] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:13,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:13,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:13,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:13,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:15,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:15,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:15,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:15,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:15,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:15,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:15,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:15,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,847] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:16,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:17,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:17,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:17,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:17,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:17,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:17,376] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:17,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:17,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:17,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:17,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:17,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:17,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,082] [WARNING] [stage3.py:2497:step] 9 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 7%|▋ | 7/98 [10:41<2:16:49, 90.21s/it] {'loss': '0.5389', 'grad_norm': '2.611', 'learning_rate': '3.333e-06', 'ppl': '1.714', 'memory/max_active (GiB)': '90.17', 'memory/max_allocated (GiB)': '90.17', 'memory/device_reserved (GiB)': '92.3', 'tokens/train_per_sec_per_gpu': '83.09', 'tokens/total': 6123584, 'tokens/trainable': 1480365, 'epoch': '0.8485'} 7%|▋ | 7/98 [10:41<2:16:49, 90.21s/it][2026-04-25 04:46:24,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:24,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:25,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:26,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:28,841] [WARNING] [stage3.py:2497:step] 8 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 7%|▋ | 7/98 [10:46<2:18:25, 91.26s/it] {'loss': '0.5389', 'grad_norm': '2.628', 'learning_rate': '3.333e-06', 'ppl': '1.714', 'memory/max_active (GiB)': '90.17', 'memory/max_allocated (GiB)': '90.17', 'memory/device_reserved (GiB)': '92.69', 'tokens/train_per_sec_per_gpu': '81.34', 'tokens/total': 6123584, 'tokens/trainable': 1480365, 'epoch': '0.8485'} 7%|▋ | 7/98 [10:46<2:18:25, 91.26s/it][2026-04-25 04:46:28,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:28,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:28,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:29,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:30,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,786] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,829] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:34,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:35,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:39,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:40,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:43,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:43,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:43,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:43,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:43,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:43,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:43,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:43,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,853] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:44,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:45,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:45,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:45,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:45,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:45,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:45,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:45,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:45,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:45,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:45,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:45,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:45,754] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:48,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:48,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:48,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:48,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:48,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:48,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:48,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:49,941] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,201] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:50,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:54,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:54,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:54,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:54,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:54,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:54,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:54,757] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:54,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:55,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:55,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:55,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:55,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:55,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:55,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:55,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:55,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:55,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:55,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,480] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,851] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:56,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:57,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:59,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:59,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:59,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:59,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:59,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:59,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:59,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:46:59,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:00,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:00,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:00,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:00,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:00,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:00,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:00,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:00,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:00,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:00,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:00,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:00,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,498] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:01,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:02,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,376] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:08,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,312] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,834] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:09,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:10,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:10,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:12,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:12,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:12,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:12,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:12,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:13,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,634] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:14,748] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:18,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:18,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:18,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:18,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:18,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,552] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,658] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:19,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:20,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:20,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:20,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:20,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:20,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:20,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:20,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:20,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:20,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:20,550] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:20,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:20,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:23,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:23,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:23,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:23,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:23,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:23,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:23,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:23,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:23,955] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,773] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:24,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:25,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:25,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:25,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:25,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:25,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:25,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:25,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:25,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:30,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:30,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:30,657] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:30,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:30,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:30,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:30,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:30,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,363] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,601] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:31,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:32,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:32,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:32,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:32,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:32,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:32,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:32,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:32,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:32,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:32,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:32,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,703] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:35,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:36,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:37,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:37,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:37,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:37,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:37,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:37,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:37,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:40,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:40,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:40,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:40,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:40,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,402] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:41,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:42,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:45,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:45,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:45,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:45,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:45,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:45,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:45,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:45,791] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:45,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:45,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:45,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:45,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:46,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:47,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:47,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:47,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:47,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:47,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:47,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:47,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:47,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:53,759] [WARNING] [stage3.py:2497:step] 10 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 8%|▊ | 8/98 [12:11<2:15:03, 90.04s/it] {'loss': '0.5077', 'grad_norm': '2.189', 'learning_rate': '3.889e-06', 'ppl': '1.661', 'memory/max_active (GiB)': '91', 'memory/max_allocated (GiB)': '91', 'memory/device_reserved (GiB)': '92.87', 'tokens/train_per_sec_per_gpu': '72.78', 'tokens/total': 6978880, 'tokens/trainable': 1690270, 'epoch': '0.9697'} 8%|▊ | 8/98 [12:11<2:15:03, 90.04s/it][2026-04-25 04:47:53,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:53,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:53,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:53,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:54,958] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,754] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:55,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:56,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:56,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:57,945] [WARNING] [stage3.py:2497:step] 9 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 8%|▊ | 8/98 [12:16<2:15:51, 90.58s/it] {'loss': '0.5077', 'grad_norm': '2.19', 'learning_rate': '3.889e-06', 'ppl': '1.661', 'memory/max_active (GiB)': '91', 'memory/max_allocated (GiB)': '91', 'memory/device_reserved (GiB)': '92.73', 'tokens/train_per_sec_per_gpu': '73.24', 'tokens/total': 6978880, 'tokens/trainable': 1690270, 'epoch': '0.9697'} 8%|▊ | 8/98 [12:16<2:15:51, 90.58s/it][2026-04-25 04:47:57,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,748] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:58,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:47:59,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:00,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:00,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:00,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:00,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:00,268] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:06,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:06,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:06,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:07,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:08,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:08,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:08,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:08,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:08,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:08,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:08,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:08,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:08,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:08,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:08,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:08,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1998124] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:10,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:10,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:10,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:10,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:10,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:10,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,268] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,500] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:11,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:12,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:12,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:12,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:12,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:12,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:12,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:12,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:12,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:12,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:12,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1706109] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-25 04:48:17,080] [WARNING] [stage3.py:2497:step] 3 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 9%|▉ | 9/98 [12:34<1:42:37, 69.18s/it] {'loss': '0.4966', 'grad_norm': '1.752', 'learning_rate': '4.444e-06', 'ppl': '1.643', 'memory/max_active (GiB)': '89.6', 'memory/max_allocated (GiB)': '89.6', 'memory/device_reserved (GiB)': '92.51', 'tokens/train_per_sec_per_gpu': '253.2', 'tokens/total': 7192256, 'tokens/trainable': 1742416, 'epoch': '1'} 9%|▉ | 9/98 [12:34<1:42:37, 69.18s/it][2026-04-25 04:48:19,368] [INFO] [axolotl.core.trainers.base._save:741] [PID:1998124] Saving model checkpoint to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v8-1000-axolotl__Qwen3-8B/checkpoint-9 Writing model shards: 0%| | 0/1 [00:00