[2026-04-22 17:08:48,040] [DEBUG] [axolotl.utils.config.resolve_dtype:74] [PID:1024775] bf16 support detected, enabling for this configuration. [2026-04-22 17:08:48,597] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:1024775] baseline 0.000GB () [2026-04-22 17:08:48,598] [INFO] [axolotl.cli.config.load_cfg:341] [PID:1024775] config: { "activation_offloading": true, "adam_beta1": 0.9, "adam_beta2": 0.95, "axolotl_config_path": "/e/scratch/jureap59/feuer1/code/axolotl_configs/qwen3_8b_sera_v3_1000.yaml", "base_model": "Qwen/Qwen3-8B", "base_model_config": "Qwen/Qwen3-8B", "batch_size": 32, "bf16": true, "capabilities": { "bf16": true, "compute_capability": "sm_90", "fp8": true, "n_gpu": 4, "n_node": 1, "tf32": true }, "chat_template": "chatml", "context_parallel_size": 1, "dataloader_num_workers": 4, "dataloader_pin_memory": true, "dataloader_prefetch_factor": 256, "dataset_num_proc": 288, "dataset_prepared_path": "/e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/sera-v3-1000", "datasets": [ { "chat_template": "tokenizer_default", "data_files": [ "sera-4.5a-full-t1_v3_1000.jsonl" ], "ds_type": "json", "field_messages": "messages", "message_field_training": "train", "message_property_mappings": { "content": "content", "role": "role" }, "path": "laion/Sera-4.5A-Full-T1-v3-1000", "trust_remote_code": false, "type": "chat_template" } ], "ddp": true, "deepspeed": { "bf16": { "enabled": true }, "gradient_accumulation_steps": "auto", "gradient_clipping": "auto", "train_batch_size": "auto", "train_micro_batch_size_per_gpu": "auto", "wall_clock_breakdown": false, "zero_optimization": { "contiguous_gradients": true, "gather_16bit_weights_on_model_save": true, "max_live_parameters": 0, "max_reuse_distance": 0, "overlap_comm": true, "reduce_bucket_size": "auto", "stage": 3, "stage3_param_persistence_threshold": "auto", "stage3_prefetch_bucket_size": "auto", "sub_group_size": 0 } }, "device": "cuda:0", "device_map": { "": 0 }, "dion_rank_fraction": 1.0, "dion_rank_multiple_of": 1, "eaft_alpha": 1.0, "eaft_k": 20, "env_capabilities": { "torch_version": "2.9.1" }, "eval_batch_size": 1, "eval_causal_lm_metrics": [ "sacrebleu", "comet", "ter", "chrf" ], "eval_max_new_tokens": 128, "eval_table_size": 0, "evals_per_epoch": 0, "experimental_skip_move_to_device": true, "flash_attention": true, "fp16": false, "generate_samples": false, "generation_do_sample": true, "generation_max_new_tokens": 50, "generation_prompt_ratio": 0.5, "generation_temperature": 0.7, "gradient_accumulation_steps": 8, "gradient_checkpointing": true, "gradient_checkpointing_kwargs": { "use_reentrant": true }, "include_tkps": true, "layer_offloading": false, "learning_rate": 1e-05, "lisa_layers_attribute": "model.layers", "load_best_model_at_end": false, "load_in_4bit": false, "load_in_8bit": false, "local_rank": 0, "logging_steps": 1, "lora_dropout": 0.0, "loraplus_lr_embedding": 1e-06, "loss_watchdog_patience": 3, "loss_watchdog_threshold": 5.0, "lr_scheduler": "cosine", "max_grad_norm": 1.0, "mean_resizing_embeddings": false, "merge_method": "memory_efficient", "micro_batch_size": 1, "model_config_type": "qwen3", "num_epochs": 3.0, "num_generation_samples": 3, "optimizer": "adamw_torch", "otel_metrics_host": "localhost", "otel_metrics_port": 8000, "output_dir": "/e/data1/datasets/playground/ot-baf/checkpoints/sera-v3-1000-axolotl__Qwen3-8B", "pretrain_multipack_attn": true, "profiler_steps_start": 0, "qlora_sharded_model_loading": false, "quantize_moe_experts": false, "ray_num_workers": 1, "resources_per_worker": { "GPU": 1 }, "sample_packing_bin_size": 200, "sample_packing_group_size": 100000, "save_only_model": false, "save_safetensors": true, "save_strategy": "epoch", "sequence_len": 32768, "shuffle_before_merging_datasets": false, "shuffle_merged_datasets": true, "skip_prepare_dataset": false, "streaming_multipack_buffer_size": 10000, "strict": false, "tensor_parallel_size": 1, "tf32": false, "tiled_mlp_use_original_mlp": true, "tokenizer_config": "Qwen/Qwen3-8B", "tokenizer_save_jinja_files": true, "torch_dtype": "torch.bfloat16", "train_on_inputs": false, "trl": { "async_prefetch": false, "log_completions": false, "mask_truncated_completions": false, "ref_model_mixup_alpha": 0.9, "ref_model_sync_steps": 64, "replay_buffer_size": 0, "replay_recompute_logps": true, "reroll_max_groups": 1, "reroll_start_fraction": 1.0, "reward_num_workers": 1, "scale_rewards": true, "skip_zero_advantage_batches": true, "sync_ref_model": false, "use_data_producer": false, "use_vllm": false, "vllm_lora_sync": false, "vllm_server_host": "0.0.0.0", "vllm_server_port": 8000 }, "use_otel_metrics": false, "use_ray": false, "val_set_size": 0.0, "vllm": { "device": "auto", "dtype": "auto", "gpu_memory_utilization": 0.9, "host": "0.0.0.0", "port": 8000 }, "wandb_name": "sera-v3-1000-axolotl__Qwen3-8B", "warmup_ratio": 0.1875, "weight_decay": 0.01, "world_size": 4 } [2026-04-22 17:08:48,619] [INFO] [axolotl.cli.checks.check_user_token:37] [PID:1024775] Skipping HuggingFace token verification because HF_HUB_OFFLINE is set to True. Only local files will be used. [2026-04-22 17:08:49,088] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:1024775] EOS: 151645 / <|im_end|> [2026-04-22 17:08:49,088] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:1024775] BOS: None / None [2026-04-22 17:08:49,088] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:1024775] PAD: 151643 / <|endoftext|> [2026-04-22 17:08:49,088] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:1024775] UNK: None / None [2026-04-22 17:08:49,200] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:475] [PID:1024775] Loading prepared dataset from disk at /e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/sera-v3-1000/0b7c53810080c53afa3bb49c397dff74... [2026-04-22 17:08:49,676] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:1024775] total_num_tokens: 8_072_292 [2026-04-22 17:08:50,363] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:1024775] `total_supervised_tokens: 1_547_775` [2026-04-22 17:08:50,363] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:1024775] total_num_steps: 29 [2026-04-22 17:08:50,363] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:1024775] Maximum number of steps set at 29 [2026-04-22 17:08:50,400] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:1024775] loading tokenizer... Qwen/Qwen3-8B [2026-04-22 17:08:50,662] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:1024775] EOS: 151645 / <|im_end|> [2026-04-22 17:08:50,662] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:1024775] BOS: None / None [2026-04-22 17:08:50,662] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:1024775] PAD: 151643 / <|endoftext|> [2026-04-22 17:08:50,662] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:1024775] UNK: None / None [2026-04-22 17:08:50,662] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:1024775] Loading model [2026-04-22 17:08:50,708] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:1024775] Patched Trainer.evaluation_loop with nanmean loss calculation [2026-04-22 17:08:50,709] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:1024775] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation [2026-04-22 17:08:50,856] [INFO] [axolotl.monkeypatch.attention.flash_attn_4.patch_flash_attn_4:52] [PID:1024775] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 [2026-04-22 17:08:50,858] [INFO] [axolotl.monkeypatch.deepspeed_utils.patch_checkpoint_wrapper_setattr:52] [PID:1024775] CheckpointWrapper patched to forward DeepSpeed attributes [2026-04-22 17:09:02,275] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:361] [PID:1024775] Converting modules to torch.bfloat16 [2026-04-22 17:09:02,279] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:1024775] Memory usage after model load 4.973GB (+4.973GB allocated, +5.080GB reserved) [2026-04-22 17:09:08,636] [INFO] [axolotl.train.save_initial_configs:421] [PID:1024775] Pre-saving tokenizer to /e/data1/datasets/playground/ot-baf/checkpoints/sera-v3-1000-axolotl__Qwen3-8B... [2026-04-22 17:09:08,757] [INFO] [axolotl.train.save_initial_configs:426] [PID:1024775] Pre-saving model config to /e/data1/datasets/playground/ot-baf/checkpoints/sera-v3-1000-axolotl__Qwen3-8B... [2026-04-22 17:09:08,760] [INFO] [axolotl.train.execute_training:222] [PID:1024775] Starting trainer... [2026-04-22 17:09:09,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:09,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_id to wrapped module Qwen3DecoderLayer 0%| | 0/29 [00:00 self.virtual_memory_safe_pct=60% of virtual memory used [2026-04-22 17:09:26,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:26,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:26,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:26,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:26,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:26,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:26,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:26,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:26,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:26,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,079] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,206] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,843] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:27,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:28,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:28,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:28,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:28,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:28,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:28,362] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:28,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:28,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:28,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:28,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:40,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:40,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:40,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:40,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:40,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:40,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:40,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:40,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:40,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:40,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:40,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:40,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:41,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:42,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:42,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:42,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:42,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:42,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:42,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:50,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:50,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:50,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:50,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:50,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,612] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:51,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,634] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:09:52,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,171] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:00,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:01,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:11,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,570] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:12,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:13,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:13,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:13,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:13,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:13,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:13,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:21,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:21,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:21,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:21,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:21,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:21,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:21,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:21,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:21,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:21,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,798] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:22,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:23,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:23,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:23,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:23,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:23,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:23,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:23,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:23,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:23,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,536] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:32,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:33,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:34,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:34,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:34,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:34,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer 3%|▎ | 1/29 [01:32<43:18, 92.80s/it] {'loss': '1.086', 'grad_norm': '18.57', 'learning_rate': '0', 'ppl': '2.961', 'memory/max_active (GiB)': '76.08', 'memory/max_allocated (GiB)': '74.92', 'memory/device_reserved (GiB)': '88.51', 'tokens/train_per_sec_per_gpu': '63.71', 'tokens/total': 866752, 'tokens/trainable': 161897, 'epoch': '0.1067'} 3%|▎ | 1/29 [01:32<43:18, 92.80s/it][2026-04-22 17:10:43,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:43,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:44,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:45,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:45,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:45,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:45,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:54,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:54,748] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:54,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:54,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:54,882] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:54,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:54,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,168] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:55,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:56,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:56,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:56,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:56,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:56,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:56,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:56,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:56,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:56,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:56,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:56,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:10:56,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:06,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:06,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:06,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:06,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:06,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:06,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:06,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:06,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:06,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,856] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:07,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:08,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:08,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:08,161] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:08,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:08,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:08,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:08,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:08,482] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:08,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:08,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:08,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:08,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:19,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,614] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:20,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:21,040] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,313] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:29,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:30,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:38,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:38,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:38,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:38,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:38,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:38,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:38,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:38,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:38,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:38,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:39,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:40,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:40,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:40,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:40,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:40,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:40,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:40,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:40,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:40,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:50,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:50,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:50,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:50,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:50,859] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:50,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:50,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:50,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,628] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,692] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:51,956] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:52,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:52,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:52,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:52,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:52,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:52,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:52,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:52,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:52,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:52,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:52,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:11:52,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:03,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:03,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:03,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:03,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:03,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:03,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:03,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:03,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:03,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,507] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:04,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:05,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:05,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:05,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:05,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:05,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:05,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:05,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:05,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:05,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:05,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:15,382] [WARNING] [stage3.py:2497:step] 6 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 7%|▋ | 2/29 [03:05<41:38, 92.52s/it] {'loss': '1.114', 'grad_norm': '19.11', 'learning_rate': '2e-06', 'ppl': '3.046', 'memory/max_active (GiB)': '84.51', 'memory/max_allocated (GiB)': '84.51', 'memory/device_reserved (GiB)': '92.83', 'tokens/train_per_sec_per_gpu': '62.7', 'tokens/total': 1715584, 'tokens/trainable': 330311, 'epoch': '0.2133'} 7%|▋ | 2/29 [03:05<41:38, 92.52s/it][2026-04-22 17:12:15,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:15,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:15,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:15,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:15,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:15,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:15,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:15,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:15,862] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:15,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:15,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:16,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:16,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:16,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:16,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:16,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:16,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:16,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:16,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:16,956] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:17,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:17,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:17,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:17,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:17,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:17,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:17,734] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:17,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:17,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:17,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:17,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:18,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:18,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:18,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:18,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:18,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:28,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:28,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:28,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:28,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:28,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:28,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:28,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:28,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:28,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:28,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:28,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,851] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:29,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:30,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:30,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:30,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:30,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:30,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:30,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:30,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:30,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:30,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,362] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:40,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:41,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:42,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:42,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:42,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:42,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:42,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:51,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:51,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:51,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:51,834] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:51,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:51,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,251] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,311] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:52,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,206] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:12:53,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,268] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,748] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:02,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,706] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:03,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:04,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:04,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:12,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:13,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:14,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:14,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:14,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:14,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,371] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:23,947] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,200] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,921] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:24,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:25,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:25,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:25,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:25,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:25,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:25,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:25,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:35,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:35,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:35,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:35,658] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:35,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:35,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:35,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:35,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:35,955] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,500] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:36,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:37,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:37,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:37,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:37,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:37,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:37,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:37,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:37,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:37,552] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:37,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,164] [WARNING] [stage3.py:2497:step] 12 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 10%|█ | 3/29 [04:36<39:56, 92.18s/it] {'loss': '1.083', 'grad_norm': '17.84', 'learning_rate': '4e-06', 'ppl': '2.953', 'memory/max_active (GiB)': '90.17', 'memory/max_allocated (GiB)': '90.17', 'memory/device_reserved (GiB)': '92.83', 'tokens/train_per_sec_per_gpu': '51.34', 'tokens/total': 2569408, 'tokens/trainable': 495726, 'epoch': '0.32'} 10%|█ | 3/29 [04:36<39:56, 92.18s/it][2026-04-22 17:13:47,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,658] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:47,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:48,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:49,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:49,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:49,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:49,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:49,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:49,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:58,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:58,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:58,631] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:58,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:58,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:58,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:58,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:58,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,251] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,319] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,734] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:13:59,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:00,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:11,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:11,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:11,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:11,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:11,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:11,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:11,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:11,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:11,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:12,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:13,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:13,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:13,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:13,200] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:13,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:13,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:13,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:13,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:13,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:13,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:13,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:23,363] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:23,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:23,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:23,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:23,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:23,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:23,702] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:23,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:23,834] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:23,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:23,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,162] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:24,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:25,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:25,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:25,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:25,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:25,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:25,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:25,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:25,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:25,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:25,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:36,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,327] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:37,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:38,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:38,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:38,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:38,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:38,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:38,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:48,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:48,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:48,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,056] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:49,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,853] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:14:50,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:00,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:00,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:00,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:01,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:02,768] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:11,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:12,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:13,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:13,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:13,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:13,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:13,268] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:23,959] [WARNING] [stage3.py:2497:step] 10 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 14%|█▍ | 4/29 [06:13<39:10, 94.00s/it] {'loss': '0.9875', 'grad_norm': '10.35', 'learning_rate': '6e-06', 'ppl': '2.685', 'memory/max_active (GiB)': '91.35', 'memory/max_allocated (GiB)': '91.35', 'memory/device_reserved (GiB)': '92.9', 'tokens/train_per_sec_per_gpu': '65.19', 'tokens/total': 3453504, 'tokens/trainable': 670131, 'epoch': '0.4267'} 14%|█▍ | 4/29 [06:13<39:10, 94.00s/it][2026-04-22 17:15:23,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,200] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:24,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:25,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:26,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:26,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:36,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:36,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:36,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:36,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:36,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:36,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:36,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:36,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:36,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:36,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:36,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:36,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:37,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:37,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:37,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:37,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:37,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:37,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:37,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:37,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:37,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:37,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:37,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,727] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:38,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,446] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:47,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,470] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:48,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:49,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:49,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:49,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:49,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:49,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:49,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,480] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,688] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:15:59,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,322] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,701] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:00,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:08,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:08,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:08,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:08,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,825] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:09,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,692] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:10,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:19,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:19,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:19,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,846] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:20,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:21,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:22,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:22,125] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:22,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:32,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:32,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:33,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,319] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:34,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:44,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:44,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:44,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:44,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:44,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:44,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:44,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:44,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,268] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:45,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:46,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:46,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:46,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:46,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:46,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:46,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:46,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:46,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:46,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:46,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:46,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:56,991] [WARNING] [stage3.py:2497:step] 10 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 17%|█▋ | 5/29 [07:46<37:27, 93.65s/it] {'loss': '0.8417', 'grad_norm': '5.001', 'learning_rate': '8e-06', 'ppl': '2.32', 'memory/max_active (GiB)': '92.18', 'memory/max_allocated (GiB)': '92.18', 'memory/device_reserved (GiB)': '92.88', 'tokens/train_per_sec_per_gpu': '48.5', 'tokens/total': 4297856, 'tokens/trainable': 832861, 'epoch': '0.5333'} 17%|█▋ | 5/29 [07:46<37:27, 93.65s/it][2026-04-22 17:16:57,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:57,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,902] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:58,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:59,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:59,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:16:59,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:08,947] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,130] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,872] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:09,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:10,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:10,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:10,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:10,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:10,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:20,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:20,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:21,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:21,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:21,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:21,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:21,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:21,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:21,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:21,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:21,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:21,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:22,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:23,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:23,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:23,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:23,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:23,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:23,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:23,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:23,511] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,655] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:34,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,522] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:35,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:36,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:36,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:36,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:36,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:36,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:36,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:36,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:46,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:46,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:46,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:46,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:46,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:46,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:46,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:46,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,590] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,717] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:47,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:48,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:48,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:48,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:48,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:48,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:48,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:48,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:48,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:48,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:48,612] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:48,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:48,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:58,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:58,791] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:58,852] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:58,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:58,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:17:59,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,830] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:00,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:01,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:11,973] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,647] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:12,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:20,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:20,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:20,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:20,695] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:20,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:20,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:20,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:20,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:20,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:20,916] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:20,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:20,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,432] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:21,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:22,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:22,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:22,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:29,392] [WARNING] [stage3.py:2497:step] 13 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 21%|██ | 6/29 [09:19<35:44, 93.23s/it] {'loss': '0.8376', 'grad_norm': '6.239', 'learning_rate': '1e-05', 'ppl': '2.311', 'memory/max_active (GiB)': '91.23', 'memory/max_allocated (GiB)': '91.23', 'memory/device_reserved (GiB)': '92.61', 'tokens/train_per_sec_per_gpu': '48.17', 'tokens/total': 5179136, 'tokens/trainable': 1009902, 'epoch': '0.64'} 21%|██ | 6/29 [09:19<35:44, 93.23s/it][2026-04-22 17:18:29,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:29,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:29,512] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:29,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:29,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:29,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:29,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:29,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:29,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:29,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:29,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,706] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:30,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:31,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:31,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:31,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:31,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:31,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:31,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:31,363] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:31,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:31,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,627] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:41,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:42,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:43,032] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:43,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:43,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:52,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:52,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:52,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:52,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:52,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:52,703] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:52,734] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:52,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:52,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:52,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:52,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:53,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:53,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:53,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:53,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:53,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:53,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:53,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:53,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:53,480] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:53,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:53,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:53,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,560] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:18:54,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:04,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:04,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:04,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:04,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:04,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:04,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:04,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:04,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:04,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:04,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:04,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:04,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,334] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:05,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:06,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:06,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:06,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:06,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:06,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:06,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:06,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:06,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:06,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:17,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:17,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:17,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:17,692] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:17,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:17,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:17,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:17,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:17,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:17,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:18,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:19,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:19,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:19,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:19,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:19,297] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:19,363] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:19,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:19,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:19,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:19,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:29,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:29,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:29,495] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:29,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:29,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:29,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:29,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:29,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:29,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:29,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:29,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:29,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,158] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,364] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:30,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:31,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:31,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:31,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:31,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:31,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:39,847] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:39,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:39,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:39,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,339] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:40,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:41,670] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:49,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,028] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:50,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:51,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:58,970] [WARNING] [stage3.py:2497:step] 8 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 24%|██▍ | 7/29 [10:48<33:44, 92.03s/it] {'loss': '0.776', 'grad_norm': '4.316', 'learning_rate': '9.957e-06', 'ppl': '2.173', 'memory/max_active (GiB)': '86.63', 'memory/max_allocated (GiB)': '86.63', 'memory/device_reserved (GiB)': '92.88', 'tokens/train_per_sec_per_gpu': '39.64', 'tokens/total': 6040320, 'tokens/trainable': 1171791, 'epoch': '0.7467'} 24%|██▍ | 7/29 [10:48<33:44, 92.03s/it][2026-04-22 17:19:58,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,268] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,327] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,569] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,872] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:19:59,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,536] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,657] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,779] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:00,962] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:01,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:01,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,594] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,634] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:10,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,145] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:11,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:12,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:12,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:20,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:20,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:20,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:20,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:20,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:20,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:20,661] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:20,716] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:20,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:20,830] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:20,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:20,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,117] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,882] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:21,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:22,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:22,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:22,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:22,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:22,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:22,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:22,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:31,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:31,830] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:31,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:31,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:31,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:32,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:32,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:32,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:32,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:32,658] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:32,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:32,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:32,829] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:32,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:32,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,360] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,754] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:33,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:34,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:34,055] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:34,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:34,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:34,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:34,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:34,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:43,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:43,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:43,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:43,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:43,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,595] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:44,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:45,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:45,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:45,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:45,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:45,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:45,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:45,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:45,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:45,484] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:45,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:45,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:45,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:54,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,482] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:55,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:56,019] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:56,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:56,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:56,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:56,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:20:56,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,864] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:07,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,362] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:08,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:09,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:17,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:17,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:17,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:17,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,511] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:18,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:19,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:29,406] [WARNING] [stage3.py:2497:step] 7 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 28%|██▊ | 8/29 [12:19<32:02, 91.53s/it] {'loss': '0.7641', 'grad_norm': '3.037', 'learning_rate': '9.83e-06', 'ppl': '2.147', 'memory/max_active (GiB)': '86.87', 'memory/max_allocated (GiB)': '86.87', 'memory/device_reserved (GiB)': '92.9', 'tokens/train_per_sec_per_gpu': '48.91', 'tokens/total': 6914432, 'tokens/trainable': 1319202, 'epoch': '0.8533'} 28%|██▊ | 8/29 [12:19<32:02, 91.53s/it][2026-04-22 17:21:29,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:29,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:29,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:29,570] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:29,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:29,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:29,728] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:29,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:29,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:29,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:29,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:29,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,322] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,376] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:30,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:31,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:31,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:31,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:31,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:31,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:31,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:38,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:38,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:38,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:38,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:38,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,206] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,498] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,558] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,678] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,859] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:39,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:40,039] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:40,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:40,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:40,219] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:40,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:40,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:40,390] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:40,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:40,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:40,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:40,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:41,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:41,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:41,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:50,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:50,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:50,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,529] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,665] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:51,938] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,811] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:52,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:53,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:53,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:53,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:21:53,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,649] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:04,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:05,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:05,063] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:05,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:05,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:05,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:05,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:05,359] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:05,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:05,825] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:05,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:05,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:05,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:06,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:06,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:06,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:06,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:06,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:06,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:06,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:06,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:16,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:16,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:16,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:16,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:16,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:16,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:16,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:16,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:16,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:16,778] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:16,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:16,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:17,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:17,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:17,440] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:17,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:17,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:17,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:17,706] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:17,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:17,840] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:17,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:17,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,511] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,646] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:18,847] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:28,634] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:28,654] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:28,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:28,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:28,754] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:28,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:28,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:28,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:28,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:28,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,033] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,643] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,714] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,767] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:29,955] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:30,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:30,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:30,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:30,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:30,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:30,368] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:30,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:30,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:30,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:30,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:30,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:30,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,242] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,612] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:42,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:43,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:44,008] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:44,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:44,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:44,206] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:44,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:54,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:54,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:54,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:54,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:54,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:54,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:54,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:54,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:54,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:55,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:56,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:56,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:56,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:56,188] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:56,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:56,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:56,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:56,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:56,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:22:56,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:06,768] [WARNING] [stage3.py:2497:step] 9 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 31%|███ | 9/29 [13:56<31:07, 93.35s/it] {'loss': '0.6519', 'grad_norm': '1.971', 'learning_rate': '9.619e-06', 'ppl': '1.919', 'memory/max_active (GiB)': '91.6', 'memory/max_allocated (GiB)': '91.6', 'memory/device_reserved (GiB)': '92.9', 'tokens/train_per_sec_per_gpu': '53.16', 'tokens/total': 7778432, 'tokens/trainable': 1487467, 'epoch': '0.96'} 31%|███ | 9/29 [13:56<31:07, 93.35s/it][2026-04-22 17:23:06,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:06,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:06,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:06,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:07,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:08,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:17,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:17,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:17,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:17,600] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:17,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:17,726] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:17,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:17,852] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:17,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:17,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:18,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:19,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:19,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:19,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:19,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:19,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:19,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:19,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:19,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:19,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:19,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:28,878] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:28,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:28,947] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:28,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,322] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,803] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,860] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:29,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,058] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:30,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:1024775] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer [2026-04-22 17:23:41,039] [WARNING] [stage3.py:2497:step] 3 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time 34%|███▍ | 10/29 [14:30<23:47, 75.11s/it] {'loss': '0.6358', 'grad_norm': '1.466', 'learning_rate': '9.33e-06', 'ppl': '1.889', 'memory/max_active (GiB)': '89.82', 'memory/max_allocated (GiB)': '89.82', 'memory/device_reserved (GiB)': '91.57', 'tokens/train_per_sec_per_gpu': '158.1', 'tokens/total': 8081344, 'tokens/trainable': 1547775, 'epoch': '1'} 34%|███▍ | 10/29 [14:30<23:47, 75.11s/it][2026-04-22 17:23:43,393] [INFO] [axolotl.core.trainers.base._save:741] [PID:1024775] Saving model checkpoint to /e/data1/datasets/playground/ot-baf/checkpoints/sera-v3-1000-axolotl__Qwen3-8B/checkpoint-10 Writing model shards: 0%| | 0/1 [00:00