[2026-04-23 08:56:21,780] [DEBUG] [axolotl.utils.config.resolve_dtype:74] [PID:916134] bf16 support detected, enabling for this configuration.
[2026-04-23 08:56:21,837] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:916134] baseline 0.000GB ()
[2026-04-23 08:56:21,837] [INFO] [axolotl.cli.config.load_cfg:341] [PID:916134] config:
{
  "activation_offloading": true,
  "adam_beta1": 0.9,
  "adam_beta2": 0.95,
  "axolotl_config_path": "/e/scratch/jureap59/feuer1/code/axolotl_configs/qwen3_8b_cf_v3_316.yaml",
  "base_model": "Qwen/Qwen3-8B",
  "base_model_config": "Qwen/Qwen3-8B",
  "batch_size": 32,
  "bf16": true,
  "capabilities": {
    "bf16": true,
    "compute_capability": "sm_90",
    "fp8": true,
    "n_gpu": 4,
    "n_node": 1,
    "tf32": true
  },
  "chat_template": "chatml",
  "context_parallel_size": 1,
  "dataloader_num_workers": 4,
  "dataloader_pin_memory": true,
  "dataloader_prefetch_factor": 256,
  "dataset_num_proc": 288,
  "dataset_prepared_path": "/e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/cf-v3-316",
  "datasets": [
    {
      "ds_type": "parquet",
      "message_property_mappings": {
        "content": "content",
        "role": "role"
      },
      "path": "laion/CoderForge-Preview-v3-316",
      "trust_remote_code": false
    }
  ],
  "ddp": true,
  "deepspeed": {
    "bf16": {
      "enabled": true
    },
    "gradient_accumulation_steps": "auto",
    "gradient_clipping": "auto",
    "train_batch_size": "auto",
    "train_micro_batch_size_per_gpu": "auto",
    "wall_clock_breakdown": false,
    "zero_optimization": {
      "contiguous_gradients": true,
      "gather_16bit_weights_on_model_save": true,
      "max_live_parameters": 0,
      "max_reuse_distance": 0,
      "overlap_comm": true,
      "reduce_bucket_size": "auto",
      "stage": 3,
      "stage3_param_persistence_threshold": "auto",
      "stage3_prefetch_bucket_size": "auto",
      "sub_group_size": 0
    }
  },
  "device": "cuda:0",
  "device_map": {
    "": 0
  },
  "dion_rank_fraction": 1.0,
  "dion_rank_multiple_of": 1,
  "eaft_alpha": 1.0,
  "eaft_k": 20,
  "env_capabilities": {
    "torch_version": "2.9.1"
  },
  "eval_batch_size": 1,
  "eval_causal_lm_metrics": [
    "sacrebleu",
    "comet",
    "ter",
    "chrf"
  ],
  "eval_max_new_tokens": 128,
  "eval_table_size": 0,
  "evals_per_epoch": 0,
  "experimental_skip_move_to_device": true,
  "flash_attention": true,
  "fp16": false,
  "generate_samples": false,
  "generation_do_sample": true,
  "generation_max_new_tokens": 50,
  "generation_prompt_ratio": 0.5,
  "generation_temperature": 0.7,
  "gradient_accumulation_steps": 8,
  "gradient_checkpointing": true,
  "gradient_checkpointing_kwargs": {
    "use_reentrant": true
  },
  "include_tkps": true,
  "layer_offloading": false,
  "learning_rate": 1e-05,
  "lisa_layers_attribute": "model.layers",
  "load_best_model_at_end": false,
  "load_in_4bit": false,
  "load_in_8bit": false,
  "local_rank": 0,
  "logging_steps": 1,
  "lora_dropout": 0.0,
  "loraplus_lr_embedding": 1e-06,
  "loss_watchdog_patience": 3,
  "loss_watchdog_threshold": 5.0,
  "lr_scheduler": "cosine",
  "max_grad_norm": 1.0,
  "mean_resizing_embeddings": false,
  "merge_method": "memory_efficient",
  "micro_batch_size": 1,
  "model_config_type": "qwen3",
  "num_epochs": 3.0,
  "num_generation_samples": 3,
  "optimizer": "adamw_torch",
  "otel_metrics_host": "localhost",
  "otel_metrics_port": 8000,
  "output_dir": "/e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-316-axolotl__Qwen3-8B",
  "pretrain_multipack_attn": true,
  "profiler_steps_start": 0,
  "qlora_sharded_model_loading": false,
  "quantize_moe_experts": false,
  "ray_num_workers": 1,
  "resources_per_worker": {
    "GPU": 1
  },
  "sample_packing_bin_size": 200,
  "sample_packing_group_size": 100000,
  "save_only_model": false,
  "save_safetensors": true,
  "save_strategy": "epoch",
  "sequence_len": 32768,
  "shuffle_before_merging_datasets": false,
  "shuffle_merged_datasets": true,
  "skip_prepare_dataset": false,
  "streaming_multipack_buffer_size": 10000,
  "strict": false,
  "tensor_parallel_size": 1,
  "tf32": false,
  "tiled_mlp_use_original_mlp": true,
  "tokenizer_config": "Qwen/Qwen3-8B",
  "tokenizer_save_jinja_files": true,
  "torch_dtype": "torch.bfloat16",
  "train_on_inputs": false,
  "trl": {
    "async_prefetch": false,
    "log_completions": false,
    "mask_truncated_completions": false,
    "ref_model_mixup_alpha": 0.9,
    "ref_model_sync_steps": 64,
    "replay_buffer_size": 0,
    "replay_recompute_logps": true,
    "reroll_max_groups": 1,
    "reroll_start_fraction": 1.0,
    "reward_num_workers": 1,
    "scale_rewards": true,
    "skip_zero_advantage_batches": true,
    "sync_ref_model": false,
    "use_data_producer": false,
    "use_vllm": false,
    "vllm_lora_sync": false,
    "vllm_server_host": "0.0.0.0",
    "vllm_server_port": 8000
  },
  "use_otel_metrics": false,
  "use_ray": false,
  "val_set_size": 0.0,
  "vllm": {
    "device": "auto",
    "dtype": "auto",
    "gpu_memory_utilization": 0.9,
    "host": "0.0.0.0",
    "port": 8000
  },
  "wandb_name": "cf-v3-316-axolotl__Qwen3-8B",
  "warmup_ratio": 0.1875,
  "weight_decay": 0.01,
  "world_size": 4
}
[2026-04-23 08:56:21,865] [INFO] [axolotl.cli.checks.check_user_token:37] [PID:916134] Skipping HuggingFace token verification because HF_HUB_OFFLINE is set to True. Only local files will be used.
[2026-04-23 08:56:22,372] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:916134] EOS: 151645 / <|im_end|>
[2026-04-23 08:56:22,373] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:916134] BOS: None / None
[2026-04-23 08:56:22,373] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:916134] PAD: 151643 / <|endoftext|>
[2026-04-23 08:56:22,373] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:916134] UNK: None / None
[2026-04-23 08:56:22,572] [INFO] [axolotl.utils.data.shared.load_preprocessed_dataset:475] [PID:916134] Loading prepared dataset from disk at /e/data1/datasets/playground/ot-baf/axolotl_dataset_cache/cf-v3-316/89dcaefec5108222b90555ccb5a9ef73...
[2026-04-23 08:56:22,579] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:420] [PID:916134] total_num_tokens: 2_531_370
[2026-04-23 08:56:22,583] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:438] [PID:916134] `total_supervised_tokens: 567_197`
[2026-04-23 08:56:22,583] [DEBUG] [axolotl.utils.trainer.calculate_total_num_steps:521] [PID:916134] total_num_steps: 9
[2026-04-23 08:56:22,583] [INFO] [axolotl.utils.data.sft._prepare_standard_dataset:121] [PID:916134] Maximum number of steps set at 9
[2026-04-23 08:56:22,620] [DEBUG] [axolotl.train.setup_model_and_tokenizer:70] [PID:916134] loading tokenizer... Qwen/Qwen3-8B
[2026-04-23 08:56:22,880] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:307] [PID:916134] EOS: 151645 / <|im_end|>
[2026-04-23 08:56:22,881] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:308] [PID:916134] BOS: None / None
[2026-04-23 08:56:22,881] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:309] [PID:916134] PAD: 151643 / <|endoftext|>
[2026-04-23 08:56:22,881] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:310] [PID:916134] UNK: None / None
[2026-04-23 08:56:22,881] [DEBUG] [axolotl.train.setup_model_and_tokenizer:81] [PID:916134] Loading model
[2026-04-23 08:56:22,923] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:94] [PID:916134] Patched Trainer.evaluation_loop with nanmean loss calculation
[2026-04-23 08:56:22,924] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:148] [PID:916134] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
[2026-04-23 08:56:23,172] [INFO] [axolotl.monkeypatch.attention.flash_attn_4.patch_flash_attn_4:52] [PID:916134] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4
[2026-04-23 08:56:23,173] [INFO] [axolotl.monkeypatch.deepspeed_utils.patch_checkpoint_wrapper_setattr:52] [PID:916134] CheckpointWrapper patched to forward DeepSpeed attributes
[2026-04-23 08:56:54,302] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:361] [PID:916134] Converting modules to torch.bfloat16
[2026-04-23 08:56:54,305] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:916134] Memory usage after model load 4.973GB (+4.973GB allocated, +5.080GB reserved)
[2026-04-23 08:57:00,717] [INFO] [axolotl.train.save_initial_configs:421] [PID:916134] Pre-saving tokenizer to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-316-axolotl__Qwen3-8B...
[2026-04-23 08:57:00,850] [INFO] [axolotl.train.save_initial_configs:426] [PID:916134] Pre-saving model config to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-316-axolotl__Qwen3-8B...
[2026-04-23 08:57:00,853] [INFO] [axolotl.train.execute_training:222] [PID:916134] Starting trainer...
[2026-04-23 08:57:01,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,221] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_external_parameters to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,245] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,251] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,253] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,254] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,259] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,267] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,268] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:01,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_id to wrapped module Qwen3DecoderLayer

  0%|          | 0/9 [00:00<?, ?it/s][2026-04-23 08:57:03,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:03,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,079] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,819] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:04,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,259] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,825] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:05,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:06,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:06,079] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,301] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,452] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,853] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,910] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:15,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,086] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,259] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:16,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:17,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:17,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:17,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:17,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:17,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:17,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:27,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:27,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:28,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,512] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,702] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:29,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:30,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:30,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,852] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:42,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,230] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,418] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,482] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,791] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:43,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:44,044] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:44,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:44,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:44,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:44,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:44,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:44,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:52,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:52,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:52,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:52,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:52,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,193] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,706] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:53,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,091] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,284] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:57:54,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:06,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:06,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:06,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:06,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:06,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:06,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:06,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:06,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:06,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:06,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,206] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,607] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,663] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,724] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:07,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:08,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:08,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:08,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:08,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:08,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:08,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:08,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:08,456] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:08,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:16,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:16,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:16,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,074] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,268] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,718] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:17,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,178] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,637] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,903] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:18,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:19,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:19,100] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:27,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,363] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,422] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,601] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,662] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:28,958] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,197] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,433] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:29,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:30,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:30,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:30,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:30,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:30,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer

 11%|█         | 1/9 [01:36<12:52, 96.52s/it]
                                             
{'loss': '0.8329', 'grad_norm': '15.89', 'learning_rate': '0', 'ppl': '2.3', 'memory/max_active (GiB)': '76.51', 'memory/max_allocated (GiB)': '75.03', 'memory/device_reserved (GiB)': '88.49', 'tokens/train_per_sec_per_gpu': '64.81', 'tokens/total': 847616, 'tokens/trainable': 189135, 'epoch': '0.3478'}

 11%|█         | 1/9 [01:36<12:52, 96.52s/it][2026-04-23 08:58:38,881] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:38,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:38,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,536] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,843] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:39,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,457] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:40,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:41,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:49,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:49,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:49,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:49,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,154] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,307] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,431] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,562] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,812] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,876] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:50,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,392] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,645] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,706] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:58:51,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:01,653] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:01,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:01,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:01,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:01,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:01,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:01,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:02,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,007] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,142] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:03,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:15,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:15,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,403] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,664] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:16,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,507] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,701] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,830] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,894] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:17,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:18,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:18,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:18,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:28,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:28,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:28,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:28,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:28,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:28,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:28,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:28,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:28,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:28,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:28,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,077] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,243] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,358] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,417] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,913] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:29,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:30,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:30,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:30,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:30,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:30,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:30,362] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:30,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:30,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:40,658] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:40,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:40,708] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:40,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:40,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:40,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:40,852] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:40,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:40,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,014] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,451] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,825] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:41,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:42,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:42,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:42,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:42,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:42,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:42,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:42,386] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:42,450] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:42,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:42,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:42,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:51,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:51,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:51,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:51,789] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:51,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:51,884] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:51,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:51,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,458] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,570] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:52,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:53,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:53,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:53,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:53,229] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:53,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:53,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:53,399] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:53,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:53,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 08:59:53,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:02,513] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:02,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:02,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:02,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:02,665] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:02,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:02,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:02,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:02,858] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:02,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:02,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,300] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,512] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,844] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:03,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:04,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:04,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:04,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:04,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:04,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:04,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:12,692] [WARNING] [stage3.py:2497:step] 7 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time

 22%|██▏       | 2/9 [03:10<11:04, 94.94s/it]
                                             
{'loss': '0.8281', 'grad_norm': '15.8', 'learning_rate': '5e-06', 'ppl': '2.289', 'memory/max_active (GiB)': '90.29', 'memory/max_allocated (GiB)': '90.29', 'memory/device_reserved (GiB)': '92.42', 'tokens/train_per_sec_per_gpu': '58.1', 'tokens/total': 1729344, 'tokens/trainable': 386305, 'epoch': '0.6957'}

 22%|██▏       | 2/9 [03:10<11:04, 94.94s/it][2026-04-23 09:00:12,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:12,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:12,862] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:13,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:13,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:13,451] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:13,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:13,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:13,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:13,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:13,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:13,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:13,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:13,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:14,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:14,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:14,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:14,262] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:14,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:14,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:14,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:14,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:14,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,275] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,477] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,613] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,751] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,821] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:15,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,583] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,758] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:26,940] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,126] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,309] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,542] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,709] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:27,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:28,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:28,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:28,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:28,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:28,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:28,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:28,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,323] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,657] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:39,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,404] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,463] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,715] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,781] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:40,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:41,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:41,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:41,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:41,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:41,292] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:41,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:50,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:50,955] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,772] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:51,976] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,435] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,633] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,698] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,765] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:52,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:00:53,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,297] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,549] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:03,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,104] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:04,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:05,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:05,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:05,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:05,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:05,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:05,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:05,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:15,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:15,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:15,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:15,787] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:15,837] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:15,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:15,935] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:15,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,138] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,391] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,702] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,912] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:16,967] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:17,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:17,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:17,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:17,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:17,237] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:17,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:17,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:17,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:17,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,333] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,704] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,773] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,835] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,897] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:26,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,347] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,467] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,780] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,847] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:27,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:28,034] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:28,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:01:37,851] [WARNING] [stage3.py:2497:step] 7 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time

 33%|███▎      | 3/9 [04:35<09:02, 90.47s/it]
                                             
{'loss': '0.7065', 'grad_norm': '9.597', 'learning_rate': '1e-05', 'ppl': '2.027', 'memory/max_active (GiB)': '88.28', 'memory/max_allocated (GiB)': '88.28', 'memory/device_reserved (GiB)': '92.9', 'tokens/train_per_sec_per_gpu': '68.42', 'tokens/total': 2534464, 'tokens/trainable': 567197, 'epoch': '1'}

 33%|███▎      | 3/9 [04:35<09:02, 90.47s/it][2026-04-23 09:01:41,941] [INFO] [axolotl.core.trainers.base._save:741] [PID:916134] Saving model checkpoint to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-316-axolotl__Qwen3-8B/checkpoint-3


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s][A

Writing model shards: 100%|██████████| 1/1 [00:06<00:00,  6.76s/it][A
Writing model shards: 100%|██████████| 1/1 [00:06<00:00,  6.76s/it]
[2026-04-23 09:02:09,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,263] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,383] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,504] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,623] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,748] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:09,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,169] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,231] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,416] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,480] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,658] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,719] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,784] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,907] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:10,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:11,029] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:11,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:11,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:20,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:20,426] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:20,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:20,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:20,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:20,626] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:20,678] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:20,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:20,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:20,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:20,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:20,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,324] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,379] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,514] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,568] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,875] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:21,957] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:22,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:22,080] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:22,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:22,203] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:22,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:22,326] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:22,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,319] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,419] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,794] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:32,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:33,942] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:34,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,085] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,162] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,289] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,750] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,817] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:42,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,347] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,415] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,482] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,614] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,813] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:43,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:44,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:44,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:44,147] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:44,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:44,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:44,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:54,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,193] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,563] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:55,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,064] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,514] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,579] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,834] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:56,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:57,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:57,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:02:57,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,317] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,357] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,477] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:07,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,003] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,050] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,293] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,438] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,536] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:08,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:15,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:15,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:15,506] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:15,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:15,601] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:15,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:15,701] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:15,748] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:15,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:15,850] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:15,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:15,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,006] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,258] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,363] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,410] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,512] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:16,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:17,023] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:17,079] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:17,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:17,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:17,248] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,424] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,479] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,701] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:25,977] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,346] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,609] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,872] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:26,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:27,002] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:27,067] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:27,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:27,199] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:27,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:27,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:27,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:37,977] [WARNING] [stage3.py:2497:step] 10 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time

 44%|████▍     | 4/9 [06:35<08:30, 102.18s/it]
                                              
{'loss': '0.555', 'grad_norm': '7.088', 'learning_rate': '9.505e-06', 'ppl': '1.742', 'memory/max_active (GiB)': '91.23', 'memory/max_allocated (GiB)': '91.23', 'memory/device_reserved (GiB)': '92.9', 'tokens/train_per_sec_per_gpu': '73.5', 'tokens/total': 3410048, 'tokens/trainable': 761742, 'epoch': '1.348'}

 44%|████▍     | 4/9 [06:35<08:30, 102.18s/it][2026-04-23 09:03:38,004] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,180] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,809] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,934] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:38,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,060] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,121] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,503] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:39,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:40,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:40,078] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:40,141] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:40,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:49,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:49,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,096] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,539] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:50,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,013] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,076] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,140] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,202] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,389] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,455] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,836] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:51,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:52,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:52,089] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:52,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:52,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:03:52,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,107] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,204] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,256] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,305] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,510] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:03,951] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,021] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,071] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,122] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,185] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,474] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,923] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:04,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:05,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:13,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:13,499] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:13,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:13,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:13,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:13,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:13,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:13,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:13,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:13,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,116] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,178] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,240] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,804] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:14,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:15,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:15,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:15,179] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:15,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:15,304] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:15,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:15,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:15,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:15,555] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:15,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:24,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:24,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,174] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,682] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:25,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,065] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,127] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,191] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,318] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,381] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,445] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:26,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:27,018] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:27,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:27,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,589] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,754] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:36,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,354] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,711] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,771] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,832] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:37,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:38,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:38,070] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:38,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:38,190] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:38,249] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:38,308] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,406] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,471] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,534] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,660] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:47,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,182] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,247] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,313] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,378] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,509] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,707] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,773] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,839] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:48,971] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:49,037] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:49,103] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:49,170] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:49,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:49,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:49,369] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:49,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:59,974] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:04:59,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,153] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,266] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,322] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,497] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,557] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,674] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,732] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,790] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:00,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,030] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,282] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,413] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,472] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,605] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,928] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:01,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:02,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,133] [WARNING] [stage3.py:2497:step] 12 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time

 56%|█████▌    | 5/9 [08:10<06:38, 99.65s/it] 
                                             
{'loss': '0.4561', 'grad_norm': '1.787', 'learning_rate': '8.117e-06', 'ppl': '1.578', 'memory/max_active (GiB)': '91', 'memory/max_allocated (GiB)': '91', 'memory/device_reserved (GiB)': '92.73', 'tokens/train_per_sec_per_gpu': '66.86', 'tokens/total': 4309696, 'tokens/trainable': 962414, 'epoch': '1.696'}

 56%|█████▌    | 5/9 [08:10<06:38, 99.65s/it][2026-04-23 09:05:13,181] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,355] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,531] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,652] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,833] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,892] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:13,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,436] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:14,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:15,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:15,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:15,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:15,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:15,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:23,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:23,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:23,975] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,189] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,407] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,465] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,521] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,578] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,634] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,692] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:24,979] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,036] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,327] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,774] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,825] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,877] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:25,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:34,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:34,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:34,525] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:34,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:34,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:34,671] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:34,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:34,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:34,820] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:34,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:34,918] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:34,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,283] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,470] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:35,961] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:36,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:36,092] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:36,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:36,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:36,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:36,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:36,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:36,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:36,571] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:36,632] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:36,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:46,622] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:46,641] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:46,690] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:46,744] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:46,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:46,854] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:46,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:46,969] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,025] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,083] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,384] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,586] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,723] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,862] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:47,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:48,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:48,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:48,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:48,277] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:48,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:48,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:48,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:48,552] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:48,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:48,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:48,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:48,829] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:05:59,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,043] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,194] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,233] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,535] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,585] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,689] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:00,986] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:01,038] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:01,087] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:01,137] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:01,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:01,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:01,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:01,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:01,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:01,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:01,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:01,539] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:01,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:08,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:08,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:08,871] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:08,919] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:08,966] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,118] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,264] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,316] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,363] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,485] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,548] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,736] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:09,982] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,110] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,362] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,552] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,615] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,743] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:10,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,143] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,178] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,216] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,294] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,370] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,489] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,546] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,608] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,861] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,924] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:20,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,365] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,427] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,491] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,553] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,618] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,870] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:21,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:22,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:22,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:06:31,791] [WARNING] [stage3.py:2497:step] 4 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time

 67%|██████▋   | 6/9 [09:29<04:37, 92.51s/it]
                                             
{'loss': '0.4101', 'grad_norm': '1.196', 'learning_rate': '6.113e-06', 'ppl': '1.507', 'memory/max_active (GiB)': '83.8', 'memory/max_allocated (GiB)': '83.8', 'memory/device_reserved (GiB)': '92.73', 'tokens/train_per_sec_per_gpu': '77.05', 'tokens/total': 5068928, 'tokens/trainable': 1134394, 'epoch': '2'}

 67%|██████▋   | 6/9 [09:29<04:37, 92.51s/it][2026-04-23 09:06:33,187] [INFO] [axolotl.core.trainers.base._save:741] [PID:916134] Saving model checkpoint to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-316-axolotl__Qwen3-8B/checkpoint-6


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s][A

Writing model shards: 100%|██████████| 1/1 [00:06<00:00,  6.00s/it][A
Writing model shards: 100%|██████████| 1/1 [00:06<00:00,  6.00s/it]
[2026-04-23 09:07:02,895] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:02,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,222] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,353] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,421] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,619] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,687] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:03,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,144] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,344] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,411] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,676] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:04,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:05,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:05,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:05,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:05,207] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,442] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,541] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,745] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,845] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:14,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,160] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,211] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,387] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,502] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,564] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,855] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:15,973] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:16,031] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:16,090] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:16,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:16,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:25,752] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:25,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:25,826] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:25,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:25,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,012] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,133] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,257] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,320] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,382] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,572] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,635] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,697] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,760] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:26,946] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,259] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,322] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,448] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,511] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,574] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,699] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,762] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,825] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:27,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,131] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,228] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,280] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,494] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,597] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,770] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,893] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:38,959] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,148] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,212] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,538] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,603] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,741] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:39,980] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:40,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:40,097] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:40,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:40,217] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:50,793] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:50,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:50,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:50,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:50,988] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,176] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,236] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,299] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,362] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,487] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,614] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,677] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,866] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,929] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:51,992] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,057] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,120] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,246] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,310] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,374] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,437] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,630] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,759] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,824] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,887] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:07:52,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:02,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:02,701] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:02,755] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:02,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:02,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:02,931] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:02,989] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,049] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,124] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,296] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,348] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,408] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,523] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,580] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,753] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,810] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:03,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:04,041] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:04,099] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:04,157] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:04,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:04,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:04,328] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:04,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:04,443] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:04,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:04,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:04,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:04,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,205] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,321] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,372] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,423] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,477] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,691] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,802] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,859] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:13,964] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,016] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,343] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,519] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,573] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,748] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,863] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:14,984] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:15,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:15,105] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:23,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:23,763] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:23,800] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:23,842] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:23,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:23,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:23,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,010] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,177] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,218] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,260] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,306] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,351] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,394] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,439] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,483] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,528] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,639] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,729] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,777] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,825] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,874] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,926] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:24,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:25,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:25,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:25,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:25,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:25,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:25,269] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:25,329] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:32,789] [WARNING] [stage3.py:2497:step] 8 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time

 78%|███████▊  | 7/9 [11:30<03:23, 101.82s/it]
                                              
{'loss': '0.383', 'grad_norm': '0.9671', 'learning_rate': '3.887e-06', 'ppl': '1.467', 'memory/max_active (GiB)': '90.29', 'memory/max_allocated (GiB)': '90.29', 'memory/device_reserved (GiB)': '92.38', 'tokens/train_per_sec_per_gpu': '61.84', 'tokens/total': 5937728, 'tokens/trainable': 1333382, 'epoch': '2.348'}

 78%|███████▊  | 7/9 [11:30<03:23, 101.82s/it][2026-04-23 09:08:32,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:32,868] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:32,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:32,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,053] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,115] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,178] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,301] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,361] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,425] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,551] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,611] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,672] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,738] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,864] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:33,990] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,048] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,114] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,175] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,239] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,488] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,554] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,617] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,678] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,807] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,932] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:34,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:43,444] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:43,462] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:43,520] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:43,581] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:43,640] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:43,703] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:43,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:43,827] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:43,891] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:43,954] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,017] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,081] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,209] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,272] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,398] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,524] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,587] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,650] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,775] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,838] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:44,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:45,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:45,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:45,150] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:45,214] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:45,278] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:45,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:45,405] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:45,469] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:45,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:45,593] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:55,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:55,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:55,575] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:55,629] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:55,684] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:55,739] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:55,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:55,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:55,905] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:55,963] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,075] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,129] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,186] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,298] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,375] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,430] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,492] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,556] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,685] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,749] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,815] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,879] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:56,943] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:57,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:57,095] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:57,155] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:57,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:57,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:57,337] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:57,402] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:57,466] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:57,532] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:08:57,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,094] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,112] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,166] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,223] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,340] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,459] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,518] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,577] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,764] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,828] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,890] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:08,953] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,011] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,073] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,135] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,192] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,250] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,527] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,547] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,601] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,713] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,825] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:09,941] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:10,000] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:10,059] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:10,119] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:10,183] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:10,241] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:10,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,273] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,291] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,336] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,516] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,566] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,600] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,636] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,683] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,783] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,834] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,886] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:20,937] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,159] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,213] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,330] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,388] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,447] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,505] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,565] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,686] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,746] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,865] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:21,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:22,045] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:22,106] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:22,165] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:22,225] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:22,285] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:22,345] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:30,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:30,922] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:30,978] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,039] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,101] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,164] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,226] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,288] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,350] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,475] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,537] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,598] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,659] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,721] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,782] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,843] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,904] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:31,965] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,027] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,271] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,332] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,454] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,515] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,576] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,638] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,700] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,761] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,822] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:32,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:33,005] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:42,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:42,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:42,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:42,584] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:42,625] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:42,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:42,705] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:42,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:42,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:42,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:42,915] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:42,970] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,052] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,113] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,234] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,295] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,356] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,420] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,482] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,545] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,606] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,668] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,730] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,792] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,920] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:43,983] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:44,046] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:44,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:44,172] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:44,235] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:44,303] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:44,366] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:44,428] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:44,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:53,883] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:53,900] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:53,939] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:53,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,068] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,152] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,215] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,401] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,468] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,602] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,673] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,805] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,869] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:54,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,022] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,084] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,146] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,208] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,274] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,342] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,544] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,616] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,679] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,747] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,814] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,899] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:55,960] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:09:56,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:06,444] [WARNING] [stage3.py:2497:step] 7 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time

 89%|████████▉ | 8/9 [13:04<01:39, 99.22s/it] 
                                             
{'loss': '0.3792', 'grad_norm': '0.8572', 'learning_rate': '1.883e-06', 'ppl': '1.461', 'memory/max_active (GiB)': '90.17', 'memory/max_allocated (GiB)': '90.17', 'memory/device_reserved (GiB)': '92.36', 'tokens/train_per_sec_per_gpu': '57.71', 'tokens/total': 6802944, 'tokens/trainable': 1526338, 'epoch': '2.696'}

 89%|████████▉ | 8/9 [13:04<01:39, 99.22s/it][2026-04-23 09:10:06,490] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:06,539] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:06,595] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:06,651] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:06,710] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:06,766] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:06,823] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:06,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:06,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:06,993] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,167] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,281] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,338] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,395] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,453] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,508] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,567] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,620] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,680] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,737] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,852] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,909] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:07,968] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:08,024] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:08,082] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:08,139] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:08,196] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:08,252] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:08,312] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:08,377] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:08,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:08,493] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,224] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,244] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,287] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,335] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,385] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,434] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,486] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,539] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,642] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,694] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,795] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,849] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,901] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:17,952] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,020] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,072] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,255] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,315] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,376] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,441] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,501] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,561] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,621] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,681] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,742] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,808] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,927] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:18,987] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:19,054] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:19,111] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:19,193] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:28,889] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:28,908] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:28,949] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:28,994] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,039] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,088] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,136] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,184] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,232] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,279] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,341] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,400] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,599] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,666] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,731] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,796] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,864] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:29,995] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,062] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,128] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,195] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,261] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,327] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,393] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,460] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,592] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,658] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,725] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,791] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,857] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,925] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:30,991] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:40,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:40,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:40,944] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:40,985] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,026] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,151] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,206] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,286] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,349] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,412] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,476] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,540] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,604] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,669] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,735] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,801] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,867] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,933] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:41,999] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,069] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,134] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,200] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,270] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,397] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,464] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,530] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,600] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,667] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,733] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,797] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,864] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,930] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:42,996] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:53,799] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:53,816] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:53,873] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:53,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:53,998] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,061] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,123] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,187] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,251] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,314] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,380] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,449] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,517] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,582] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,648] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,712] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,776] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,841] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,906] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:54,975] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,042] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,108] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,173] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,238] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,302] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,367] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,432] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,496] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,559] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,624] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,693] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,756] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,818] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,885] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:55,948] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:10:56,009] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:05,880] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:05,898] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:05,945] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:05,997] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,051] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,102] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,156] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,210] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,319] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,373] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,429] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,481] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,588] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,644] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,696] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,769] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,831] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,888] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:06,950] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,015] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,093] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,149] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,220] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,276] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,347] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,409] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,473] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,533] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,596] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,656] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,720] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,785] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,848] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:07,911] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:17,896] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:17,914] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:17,972] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,035] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,098] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,163] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,227] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,290] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,352] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,414] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,478] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,543] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,610] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,675] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,740] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,806] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,872] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:18,936] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,001] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,066] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,132] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,198] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,265] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,331] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,396] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,461] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,526] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,591] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,657] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,722] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,788] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,852] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,917] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:19,981] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:20,047] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:20,109] [DEBUG] [axolotl.monkeypatch.deepspeed_utils.new_setattr:43] [PID:916134] Forwarded ds_grads_remaining to wrapped module Qwen3DecoderLayer
[2026-04-23 09:11:30,596] [WARNING] [stage3.py:2497:step] 8 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time

100%|██████████| 9/9 [14:28<00:00, 94.51s/it]
                                             
{'loss': '0.3929', 'grad_norm': '0.7735', 'learning_rate': '4.952e-07', 'ppl': '1.481', 'memory/max_active (GiB)': '90.77', 'memory/max_allocated (GiB)': '90.77', 'memory/device_reserved (GiB)': '92.49', 'tokens/train_per_sec_per_gpu': '69.83', 'tokens/total': 7603392, 'tokens/trainable': 1701591, 'epoch': '3'}

100%|██████████| 9/9 [14:28<00:00, 94.51s/it][2026-04-23 09:11:33,374] [INFO] [axolotl.core.trainers.base._save:741] [PID:916134] Saving model checkpoint to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-316-axolotl__Qwen3-8B/checkpoint-9


Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s][A

Writing model shards: 100%|██████████| 1/1 [00:05<00:00,  5.72s/it][A
Writing model shards: 100%|██████████| 1/1 [00:05<00:00,  5.72s/it]

                                             
{'train_runtime': '892.4', 'train_samples_per_second': '0.323', 'train_steps_per_second': '0.01', 'train_loss': '0.5493', 'memory/max_active (GiB)': '31.77', 'memory/max_allocated (GiB)': '31.77', 'memory/device_reserved (GiB)': '92.49', 'epoch': '3', 'tokens/train_per_sec_per_gpu': '0'}

100%|██████████| 9/9 [14:52<00:00, 94.51s/it]
100%|██████████| 9/9 [14:52<00:00, 99.15s/it]
[2026-04-23 09:11:55,574] [INFO] [axolotl.train.save_trained_model:241] [PID:916134] Training completed! Saving trained model to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-316-axolotl__Qwen3-8B.
[2026-04-23 09:11:56,968] [INFO] [axolotl.core.trainers.base._save:741] [PID:916134] Saving model checkpoint to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-316-axolotl__Qwen3-8B

Writing model shards:   0%|          | 0/1 [00:00<?, ?it/s]
Writing model shards: 100%|██████████| 1/1 [00:05<00:00,  5.94s/it]
Writing model shards: 100%|██████████| 1/1 [00:05<00:00,  5.94s/it]
[2026-04-23 09:12:03,086] [INFO] [axolotl.train.save_trained_model:355] [PID:916134] Model successfully saved to /e/data1/datasets/playground/ot-baf/checkpoints/cf-v3-316-axolotl__Qwen3-8B
